diff --git a/packages/angular/build/src/builders/application/build-action.ts b/packages/angular/build/src/builders/application/build-action.ts
index 299d7cf88ef4..643115c38614 100644
--- a/packages/angular/build/src/builders/application/build-action.ts
+++ b/packages/angular/build/src/builders/application/build-action.ts
@@ -158,7 +158,10 @@ export async function* runEsBuildBuildAction(
       // Clear removed files from current watch files
       changes.removed.forEach((removedPath) => currentWatchFiles.delete(removedPath));
 
-      resetSassWorkerPoolCaches();
+      const hasSassChanges = changes.all.some((f) => /\.(scss|sass|css)$/i.test(f));
+      if (hasSassChanges) {
+        resetSassWorkerPoolCaches();
+      }
 
       const rebuildState = result.createRebuildState(changes);
       result = await withProgress('Changes detected. Rebuilding...', () => action(rebuildState));