[WIP] Trying to fix issue

.github/workflows/presubmit.yml

@@ -9,36 +9,6 @@ on:
       - main
 
 jobs:
-  build-desktop-mac:
-    name: build-mac
-    runs-on: macos-14-xlarge
-    steps:
-      - uses: actions/checkout@v4.1.6
-        with:
-          fetch-depth: 0
-      - uses: ./.github/actions/mac-prereq
-      - name: Run build script
-        run: |
-          cd build/mac && printf "y" | ./build.sh presubmit
-      - name: Test material parser
-        run: |
-          out/cmake-release/filament/test/test_material_parser
-
-  build-desktop-linux:
-    name: build-linux
-    runs-on: ubuntu-22.04-16core
-    steps:
-      - uses: actions/checkout@v4.1.6
-        with:
-          fetch-depth: 0
-      - uses: ./.github/actions/linux-prereq
-      - name: Run build script
-        run: |
-          cd build/linux && printf "y" | ./build.sh presubmit
-      - name: Test material parser
-        run: |
-          out/cmake-release/filament/test/test_material_parser
-
   build-windows:
     name: build-windows
     runs-on: windows-2022-32core
@@ -48,152 +18,194 @@ jobs:
           fetch-depth: 0
       - name: Run build script
         run: |
-          build\windows\build-github.bat presubmit
+          build\windows\build-github.bat continuous
         shell: cmd
 
-  build-android:
+  # build-desktop-mac:
-    name: build-android
+  #   name: build-mac
-    runs-on: 'ubuntu-24.04-16core'
+  #   runs-on: macos-14-xlarge
-    steps:
+  #   steps:
-      - uses: actions/checkout@v4.1.6
+  #     - uses: actions/checkout@v4.1.6
-        with:
+  #       with:
-          fetch-depth: 0
+  #         fetch-depth: 0
-      - uses: ./.github/actions/linux-prereq
+  #     - uses: ./.github/actions/mac-prereq
-      - uses: actions/setup-java@v3
+  #     - name: Run build script
-        with:
+  #       run: |
-          distribution: 'temurin'
+  #         cd build/mac && printf "y" | ./build.sh presubmit
-          java-version: '17'
+  #     - name: Test material parser
-      - name: Run build script
+  #       run: |
-        # Only build 1 64 bit target during presubmit to cut down build times during presubmit
+  #         out/cmake-release/filament/test/test_material_parser
-        # Continuous builds will build everything
-        run: |
-          cd build/android && printf "y" | ./build.sh presubmit arm64-v8a
 
-  build-ios:
+  # build-desktop-linux:
-    name: build-iOS
+  #   name: build-linux
-    runs-on: macos-14-xlarge
+  #   runs-on: ubuntu-22.04-16core
-    steps:
+  #   steps:
-      - uses: actions/checkout@v4.1.6
+  #     - uses: actions/checkout@v4.1.6
-        with:
+  #       with:
-          fetch-depth: 0
+  #         fetch-depth: 0
-      - uses: ./.github/actions/mac-prereq
+  #     - uses: ./.github/actions/linux-prereq
-      - name: Run build script
+  #     - name: Run build script
-        run: |
+  #       run: |
-          cd build/ios && printf "y" | ./build.sh presubmit
+  #         cd build/linux && printf "y" | ./build.sh presubmit
-      - name: Build iOS samples
+  #     - name: Test material parser
-        run: |
+  #       run: |
-          cd build/ios && ./build-samples.sh presubmit
+  #         out/cmake-release/filament/test/test_material_parser
 
-  build-web:
+  # build-windows:
-    name: build-web
+  #   name: build-windows
-    runs-on: 'ubuntu-24.04-16core'
+  #   runs-on: windows-2022-32core
-    steps:
+  #   steps:
-      - uses: actions/checkout@v4.1.6
+  #     - uses: actions/checkout@v4.1.6
-        with:
+  #       with:
-          fetch-depth: 0
+  #         fetch-depth: 0
-      - uses: ./.github/actions/linux-prereq
+  #     - name: Run build script
-      - uses: ./.github/actions/web-prereq
+  #       run: |
-      - name: Run build script
+  #         build\windows\build-github.bat presubmit
-        run: |
+  #       shell: cmd
-          cd build/web && printf "y" | ./build.sh presubmit
 
-  validate-docs:
+  # build-android:
-    name: validate-docs
+  #   name: build-android
-    runs-on: 'ubuntu-24.04-4core'
+  #   runs-on: 'ubuntu-24.04-16core'
-    steps:
+  #   steps:
-      - uses: actions/checkout@v4.1.6
+  #     - uses: actions/checkout@v4.1.6
-        with:
+  #       with:
-          fetch-depth: 0
+  #         fetch-depth: 0
-      - id: get_commit_msg
+  #     - uses: ./.github/actions/linux-prereq
-        uses: ./.github/actions/get-commit-msg
+  #     - uses: actions/setup-java@v3
-      - name: Check for manual edits to /docs
+  #       with:
-        run: |
+  #         distribution: 'temurin'
-          COMMIT_ID=$(echo "${{ steps.get_commit_msg.outputs.msg }}" | head -n 1 | sed "s/commit //g")
+  #         java-version: '17'
-          bash docs_src/build/presubmit_check.sh ${COMMIT_ID}
+  #     - name: Run build script
+  #       # Only build 1 64 bit target during presubmit to cut down build times during presubmit
+  #       # Continuous builds will build everything
+  #       run: |
+  #         cd build/android && printf "y" | ./build.sh presubmit arm64-v8a
 
-  test-renderdiff:
+  # build-ios:
-    name: test-renderdiff
+  #   name: build-iOS
-    runs-on: macos-14-xlarge
+  #   runs-on: macos-14-xlarge
-    steps:
+  #   steps:
-      - uses: actions/checkout@v4.1.6
+  #     - uses: actions/checkout@v4.1.6
-        with:
+  #       with:
-          fetch-depth: 0
+  #         fetch-depth: 0
-      - id: get_commit_msg
+  #     - uses: ./.github/actions/mac-prereq
-        uses: ./.github/actions/get-commit-msg
+  #     - name: Run build script
-      - uses: ./.github/actions/mac-prereq
+  #       run: |
-      - uses: ./.github/actions/get-mesa
+  #         cd build/ios && printf "y" | ./build.sh presubmit
-      - name: Prerequisites
+  #     - name: Build iOS samples
-        run: |
+  #       run: |
-          pip install tifffile numpy
+  #         cd build/ios && ./build-samples.sh presubmit
-        shell: bash
-      - name: Render and compare
-        id: render_compare
-        run: |
-          TEST_DIR=test/renderdiff
-          source ${TEST_DIR}/src/preamble.sh
-          start_
-          GOLDEN_BRANCH=$(echo "${{ steps.get_commit_msg.outputs.msg }}" | python3 ${TEST_DIR}/src/commit_msg.py)
-          bash ${TEST_DIR}/generate.sh && \
-              python3 ${TEST_DIR}/src/golden_manager.py \
-                  --branch=${GOLDEN_BRANCH} \
-                  --output=${GOLDEN_OUTPUT_DIR}
 
-          # Note that we need to upload the output even if comparison fails, so we undo `set -ex`
+  # build-web:
-          end_
+  #   name: build-web
+  #   runs-on: 'ubuntu-24.04-16core'
+  #   steps:
+  #     - uses: actions/checkout@v4.1.6
+  #       with:
+  #         fetch-depth: 0
+  #     - uses: ./.github/actions/linux-prereq
+  #     - uses: ./.github/actions/web-prereq
+  #     - name: Run build script
+  #       run: |
+  #         cd build/web && printf "y" | ./build.sh presubmit
 
-          python3 ${TEST_DIR}/src/compare.py \
+  # validate-docs:
-              --src=${GOLDEN_OUTPUT_DIR} \
+  #   name: validate-docs
-              --dest=${RENDER_OUTPUT_DIR} \
+  #   runs-on: 'ubuntu-24.04-4core'
-              --out=${DIFF_OUTPUT_DIR} 2>&1 | tee compare_output.txt
+  #   steps:
+  #     - uses: actions/checkout@v4.1.6
+  #       with:
+  #         fetch-depth: 0
+  #     - id: get_commit_msg
+  #       uses: ./.github/actions/get-commit-msg
+  #     - name: Check for manual edits to /docs
+  #       run: |
+  #         COMMIT_ID=$(echo "${{ steps.get_commit_msg.outputs.msg }}" | head -n 1 | sed "s/commit //g")
+  #         bash docs_src/build/presubmit_check.sh ${COMMIT_ID}
 
-          if grep "Failed" compare_output.txt > /dev/null; then
+  # test-renderdiff:
-             DELIMITER="EOF_FILE_CONTENT_$(date +%s)" # Using timestamp to make it more unique
+  #   name: test-renderdiff
-             echo "err<<$DELIMITER" >> "$GITHUB_OUTPUT"
+  #   runs-on: macos-14-xlarge
-             cat compare_output.txt >> "$GITHUB_OUTPUT"
+  #   steps:
-             echo "$DELIMITER" >> "$GITHUB_OUTPUT"
+  #     - uses: actions/checkout@v4.1.6
-          fi
+  #       with:
-      - uses: actions/upload-artifact@v4
+  #         fetch-depth: 0
-        with:
+  #     - id: get_commit_msg
-          name: presubmit-renderdiff-result
+  #       uses: ./.github/actions/get-commit-msg
-          path: ./out/renderdiff
+  #     - uses: ./.github/actions/mac-prereq
-      - name: Compare result
+  #     - uses: ./.github/actions/get-mesa
-        run: |
+  #     - name: Prerequisites
-          ERROR_STR="${{ steps.render_compare.outputs.err }}"
+  #       run: |
-          if [ -n "${ERROR_STR}" ]; then
+  #         pip install tifffile numpy
-              echo "${ERROR_STR}"
+  #       shell: bash
-              exit 1
+  #     - name: Render and compare
-          fi
+  #       id: render_compare
+  #       run: |
+  #         TEST_DIR=test/renderdiff
+  #         source ${TEST_DIR}/src/preamble.sh
+  #         start_
+  #         GOLDEN_BRANCH=$(echo "${{ steps.get_commit_msg.outputs.msg }}" | python3 ${TEST_DIR}/src/commit_msg.py)
+  #         bash ${TEST_DIR}/generate.sh && \
+  #             python3 ${TEST_DIR}/src/golden_manager.py \
+  #                 --branch=${GOLDEN_BRANCH} \
+  #                 --output=${GOLDEN_OUTPUT_DIR}
 
-  validate-wgsl-webgpu:
+  #         # Note that we need to upload the output even if comparison fails, so we undo `set -ex`
-    name: validate-wgsl-webgpu
+  #         end_
-    runs-on: 'ubuntu-24.04-8core'
-    steps:
-      - uses: actions/checkout@v4.1.6
-        with:
-          fetch-depth: 0
-      - uses: ./.github/actions/linux-prereq
-      - name: Run build script
-        run: ./build.sh -W debug test_filamat filament
-      - name: Run test
-        run: ./out/cmake-debug/libs/filamat/test_filamat --gtest_filter=MaterialCompiler.Wgsl*
 
-  test-code-correctness:
+  #         python3 ${TEST_DIR}/src/compare.py \
-    name: test-code-correctness
+  #             --src=${GOLDEN_OUTPUT_DIR} \
-    runs-on: 'macos-14-xlarge'
+  #             --dest=${RENDER_OUTPUT_DIR} \
-    steps:
+  #             --out=${DIFF_OUTPUT_DIR} 2>&1 | tee compare_output.txt
-      - uses: actions/checkout@v4.1.6
+
-        with:
+  #         if grep "Failed" compare_output.txt > /dev/null; then
-          fetch-depth: 0
+  #            DELIMITER="EOF_FILE_CONTENT_$(date +%s)" # Using timestamp to make it more unique
-      - uses: ./.github/actions/mac-prereq
+  #            echo "err<<$DELIMITER" >> "$GITHUB_OUTPUT"
-      - name: Install clang-tidy and deps
+  #            cat compare_output.txt >> "$GITHUB_OUTPUT"
-        run: |
+  #            echo "$DELIMITER" >> "$GITHUB_OUTPUT"
-          pip install pyyaml
+  #         fi
-          brew install llvm@${GITHUB_LLVM_VERSION}
+  #     - uses: actions/upload-artifact@v4
-          sudo ln -s "$(brew --prefix llvm)@${GITHUB_LLVM_VERSION}/bin/clang-tidy" "/usr/local/bin/clang-tidy"
+  #       with:
-      - name: Run build script
+  #         name: presubmit-renderdiff-result
-        # We need to build before clang-tidy can run analysis
+  #         path: ./out/renderdiff
-        run: |
+  #     - name: Compare result
-          # This will build for all three desktop backends on mac
+  #       run: |
-          ./build.sh -p desktop debug gltf_viewer
+  #         ERROR_STR="${{ steps.render_compare.outputs.err }}"
-      - name: Run test
+  #         if [ -n "${ERROR_STR}" ]; then
-        run: bash test/code-correctness/test.sh
+  #             echo "${ERROR_STR}"
+  #             exit 1
+  #         fi
+
+  # validate-wgsl-webgpu:
+  #   name: validate-wgsl-webgpu
+  #   runs-on: 'ubuntu-24.04-8core'
+  #   steps:
+  #     - uses: actions/checkout@v4.1.6
+  #       with:
+  #         fetch-depth: 0
+  #     - uses: ./.github/actions/linux-prereq
+  #     - name: Run build script
+  #       run: ./build.sh -W debug test_filamat filament
+  #     - name: Run test
+  #       run: ./out/cmake-debug/libs/filamat/test_filamat --gtest_filter=MaterialCompiler.Wgsl*
+
+  # test-code-correctness:
+  #   name: test-code-correctness
+  #   runs-on: 'macos-14-xlarge'
+  #   steps:
+  #     - uses: actions/checkout@v4.1.6
+  #       with:
+  #         fetch-depth: 0
+  #     - uses: ./.github/actions/mac-prereq
+  #     - name: Install clang-tidy and deps
+  #       run: |
+  #         pip install pyyaml
+  #         brew install llvm@${GITHUB_LLVM_VERSION}
+  #         sudo ln -s "$(brew --prefix llvm)@${GITHUB_LLVM_VERSION}/bin/clang-tidy" "/usr/local/bin/clang-tidy"
+  #     - name: Run build script
+  #       # We need to build before clang-tidy can run analysis
+  #       run: |
+  #         # This will build for all three desktop backends on mac
+  #         ./build.sh -p desktop debug gltf_viewer
+  #     - name: Run test
+  #       run: bash test/code-correctness/test.sh

build/windows/build-github.bat

@@ -135,7 +135,9 @@ set build_flags=-j %NUMBER_OF_PROCESSORS%
 :: cmake --build . --target sample-resources --config %config% %build_flags% || exit /b
 :: cmake --build . --target suzanne-resources --config %config% %build_flags% || exit /b
 
-cmake --build . %INSTALL% --config %config% %build_flags% -- /m || exit /b
+cmake --build . --target sample-resources --config %config% %build_flags% -- /m || exit /b
+
+:: cmake --build . %INSTALL% --config %config% %build_flags% -- /m || exit /b
 @echo off
 
 echo Disk info after building variant: %variant%

libs/filamat/src/MaterialBuilder.cpp

@@ -1165,23 +1165,30 @@ bool MaterialBuilder::generateShaders(JobSystem& jobSystem, const std::vector<Va
     std::sort(metalEntries.begin(), metalEntries.end(), compare);
     std::sort(wgslEntries.begin(), wgslEntries.end(), compare);
 
+
+    utils::slog.e <<"----------------------- a" << utils::io::endl;
     // Generate the dictionaries.
     for (const auto& s : glslEntries) {
         textDictionary.addText(s.shader);
     }
+    utils::slog.e <<"----------------------- b" << utils::io::endl;    
     for (const auto& s : essl1Entries) {
         textDictionary.addText(s.shader);
     }
+    utils::slog.e <<"----------------------- c" << utils::io::endl;        
     for (auto& s : spirvEntries) {
         std::vector const spirv{ std::move(s.data) };
         s.dictionaryIndex = spirvDictionary.addBlob(spirv);
     }
+    utils::slog.e <<"----------------------- d" << utils::io::endl;        
     for (const auto& s : metalEntries) {
         textDictionary.addText(s.shader);
     }
+    utils::slog.e <<"----------------------- e" << utils::io::endl;            
     for (const auto& s : wgslEntries) {
         textDictionary.addText(s.shader);
     }
+    utils::slog.e <<"----------------------- f" << utils::io::endl;                
 
     // Emit dictionary chunk (TextDictionaryReader and DictionaryTextChunk)
     const auto& dictionaryChunk = container.push<DictionaryTextChunk>(

libs/filamat/src/eiff/LineDictionary.cpp

@@ -30,13 +30,34 @@
 #include <string_view>
 #include <utility>
 #include <vector>
+#include <algorithm>
 
 namespace filamat {
 
 namespace {
-    bool isWordChar(char const c) {
+bool isWordChar(char const c) { return std::isalnum(c) || c == '_'; }
-        return std::isalnum(c) || c == '_';
+
+
+// Function to trim specified characters from both ends of a string_view
+std::string_view trim(std::string_view sv, std::string_view chars_to_trim = " \t\r\n") {
+    // Find the first character that is not in our set of trim characters
+    const auto start = sv.find_first_not_of(chars_to_trim);
+
+    // If no such character exists, the string is empty or all trim characters
+    if (start == std::string_view::npos) {
+        return ""; // Return an empty view
     }
+
+    // Find the last character that is not in our set of trim characters
+    const auto end = sv.find_last_not_of(chars_to_trim);
+
+    // Calculate the length of the new view
+    const auto len = end - start + 1;
+
+    // Return a new view representing the trimmed string
+    return sv.substr(start, len);
+}
+
 } // anonymous namespace
 
 LineDictionary::LineDictionary() = default;
@@ -64,10 +85,15 @@ std::vector<LineDictionary::index_t> LineDictionary::getIndices(
 }
 
 void LineDictionary::addText(std::string_view const text) noexcept {
+    utils::slog.e <<"begin addText l(" << text.length() << ")=" <<
+            trim(text.substr(0, std::min(15, (int)text.length())));
+
     size_t cur = 0;
     size_t const len = text.length();
     const char* s = text.data();
+    size_t trueCount = 0;
     while (cur < len) {
+        trueCount++;
         // Start of the current line
         size_t const pos = cur;
         // Find the end of the current line or end of text
@@ -79,12 +105,22 @@ void LineDictionary::addText(std::string_view const text) noexcept {
             cur++;
         }
         addLine({ s + pos, cur - pos });
+        if (trueCount > 1000000 && trueCount > 0) {
+            utils::slog.e <<"print addText at=" << trim(text.substr(0, std::min(15, (int)text.length()))) <<
+                    " count=" << trueCount << " cur=" << cur << utils::io::endl;
+            trueCount = 0;
+        }
     }
+    utils::slog.e <<" ----- end addText" << utils::io::endl;
 }
 
 void LineDictionary::addLine(std::string_view const line) noexcept {
+    utils::slog.e <<"********** begin addline l(" << line.length() << ")=" <<
+            trim(line.substr(0, std::min(15, (int)line.length())));
+    
     auto const lines = splitString(line);
     for (std::string_view const& subline : lines) {
+        utils::slog.e <<"addline indices=" << mLineIndices.size() << utils::io::endl;    
         // Never add a line twice.
         auto pos = mLineIndices.find(subline);
         if (pos != mLineIndices.end()) {
@@ -97,6 +133,7 @@ void LineDictionary::addLine(std::string_view const line) noexcept {
                     .index = index_t(mStrings.size() - 1),
                     .count = 1 });
     }
+    utils::slog.e <<"*********** end addline" << utils::io::endl;    
 }
 
 std::string_view LineDictionary::ltrim(std::string_view s) {
@@ -110,14 +147,26 @@ std::pair<size_t, size_t> LineDictionary::findPattern(
     // Patterns are ordered from longest to shortest to ensure correct prefix matching.
     static constexpr std::string_view kPatterns[] = { "hp_copy_", "mp_copy_", "_" };
 
+    utils::slog.e <<"begin findPattern l(" << line.length() << ")=" <<
+            trim(line.substr(0, std::min(15, (int)line.length())));
+
+    if (line.length() == 2) {
+        utils::slog.e <<"begin findPattern l[0]=" << (int) line[0] << " l[1]=" << (int) line[1] <<
+                
+                utils::io::endl;
+
+    }
+
+    size_t trueCount = 1;
     const size_t line_len = line.length();
-    for (size_t i = offset; i < line_len; ++i) {
+    for (size_t i = offset; i < line_len; ++i, ++trueCount) {
         // A pattern must be a whole word (or at the start of the string).
         if (i > 0 && isWordChar(line[i - 1])) {
             continue;
         }
 
         for (const auto& prefix : kPatterns) {
+            trueCount++;
             if (line.size() - i >= prefix.size() && line.substr(i, prefix.size()) == prefix) {
                 // A known prefix has been matched. Now, check for a sequence of digits.
                 size_t const startOfDigits = i + prefix.size();
@@ -141,11 +190,19 @@ std::pair<size_t, size_t> LineDictionary::findPattern(
                 break;
             }
         }
+        if (trueCount % 10000 == 0 && trueCount > 0) {
+            utils::slog.e <<"print findPattern at=" << trim(line) <<
+                    " count=" << trueCount << utils::io::endl;
+        }
     }
+    utils::slog.e <<" ----- end findPattern" << utils::io::endl;
     return { std::string_view::npos, 0 }; // No pattern found
 }
 
 std::vector<std::string_view> LineDictionary::splitString(std::string_view const line) {
+    utils::slog.e <<"begin splitString l=" << trim(line.substr(0,
+                    std::min(15, (int)line.length())));
+
     std::vector<std::string_view> result;
     size_t current_pos = 0;
 
@@ -154,7 +211,9 @@ std::vector<std::string_view> LineDictionary::splitString(std::string_view const
         return result;
     }
 
+    size_t trueCount = 0;
     while (current_pos < line.length()) {
+        trueCount++;
         auto const [match_pos, match_len] = findPattern(line, current_pos);
 
         if (match_pos == std::string_view::npos) {
@@ -173,7 +232,13 @@ std::vector<std::string_view> LineDictionary::splitString(std::string_view const
 
         // Move cursor past the match.
         current_pos = match_pos + match_len;
+
+        if (trueCount % 10000 == 0 && trueCount > 0) {
+            utils::slog.e <<"print splitString at=" << trim(line) <<
+                    " count=" << trueCount << utils::io::endl;
+        }
     }
+    utils::slog.e <<"------- end splitString" << utils::io::endl;
 
     return result;
 }

tools/matc/src/matc/MaterialCompiler.cpp

@@ -76,7 +76,7 @@ bool MaterialCompiler::run(const matp::Config& config) {
         return true;
     }
 
-    JobSystem js;
+    JobSystem js(1);
     js.adopt();
 
     // Write builder.build() to output.