diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go index 1ae31ff31ee..e18dec2fed4 100644 --- a/internal/arduino/builder/builder.go +++ b/internal/arduino/builder/builder.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/compilation" @@ -330,7 +331,7 @@ func (b *Builder) preprocess() error { if b.logger.VerbosityLevel() == logger.VerbosityVerbose { b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries.")) } - if err := b.librariesBuildPath.RemoveAll(); err != nil { + if err := b.removeBuildPathExecptLibsdiscoveryFiles(b.librariesBuildPath); err != nil { return err } } @@ -543,3 +544,28 @@ func (b *Builder) execCommand(command *paths.Process) error { return command.Wait() } + +func (b *Builder) removeBuildPathExecptLibsdiscoveryFiles(pathToRemove *paths.Path) error { + filesToRemove, err := pathToRemove.ReadDirRecursiveFiltered(nil, + paths.FilterOutDirectories(), + paths.FilterOutSuffixes(".libsdetect.d")) + if err != nil { + return err + } + for _, f := range filesToRemove { + if err := f.Remove(); err != nil { + return err + } + } + + dirsToRemove, err := pathToRemove.ReadDirRecursiveFiltered(nil, paths.FilterDirectories()) + if err != nil { + return err + } + // Remove directories in reverse order (deepest first) + sort.Slice(dirsToRemove, func(i, j int) bool { return len(dirsToRemove[i].String()) > len(dirsToRemove[j].String()) }) + for _, d := range dirsToRemove { + _ = d.Remove() + } + return nil +} diff --git a/internal/arduino/builder/internal/detector/cache.go b/internal/arduino/builder/internal/detector/cache.go index 62b3d355678..6db636fb39b 100644 --- a/internal/arduino/builder/internal/detector/cache.go +++ b/internal/arduino/builder/internal/detector/cache.go @@ -21,6 +21,7 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" "github.com/arduino/go-paths-helper" + "github.com/sirupsen/logrus" ) type detectorCache struct { @@ -30,7 +31,7 @@ type detectorCache struct { type detectorCacheEntry struct { AddedIncludePath *paths.Path `json:"added_include_path,omitempty"` - Compile *sourceFile `json:"compile,omitempty"` + Compile sourceFile `json:"compile,omitempty"` CompileTask *runner.Task `json:"compile_task,omitempty"` MissingIncludeH *string `json:"missing_include_h,omitempty"` } @@ -39,7 +40,7 @@ func (e *detectorCacheEntry) String() string { if e.AddedIncludePath != nil { return "Added include path: " + e.AddedIncludePath.String() } - if e.Compile != nil && e.CompileTask != nil { + if e.CompileTask != nil { return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String() } if e.MissingIncludeH != nil { @@ -51,6 +52,13 @@ func (e *detectorCacheEntry) String() string { return "No operation" } +func (e *detectorCacheEntry) LogMsg() string { + if e.CompileTask == nil { + return e.String() + } + return "Compiling: " + e.Compile.SourcePath.String() +} + func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool { return e.String() == entry.String() } @@ -94,10 +102,15 @@ func (c *detectorCache) Expect(entry *detectorCacheEntry) { if c.entries[c.curr].Equals(entry) { // Cache hit, move to the next entry c.curr++ + logrus.Tracef("[LD] CACHE: HIT %s", entry.LogMsg()) return } // Cache mismatch, invalidate and cut the remainder of the cache + logrus.Tracef("[LD] CACHE: INVALIDATE %s", entry.LogMsg()) + logrus.Tracef("[LD] (was %s)", c.entries[c.curr]) c.entries = c.entries[:c.curr] + } else { + logrus.Tracef("[LD] CACHE: MISSING %s", entry.LogMsg()) } c.curr++ c.entries = append(c.entries, entry) diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index 0a724cfe948..59d5474a8b2 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -41,6 +41,7 @@ import ( "github.com/arduino/arduino-cli/internal/i18n" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" + "github.com/sirupsen/logrus" ) type libraryResolutionResult struct { @@ -140,6 +141,7 @@ func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { // addAndBuildLibrary adds the given library to the imported libraries list and queues its source files // for further processing. func (l *SketchLibrariesDetector) addAndBuildLibrary(sourceFileQueue *uniqueSourceFileQueue, librariesBuildPath *paths.Path, library *libraries.Library) { + logrus.Tracef("[LD] LIBRARY: %s", library.Name) l.importedLibraries = append(l.importedLibraries, library) if library.Precompiled && library.PrecompiledWithSources { // Fully precompiled libraries should have no dependencies to avoid ABI breakage @@ -202,6 +204,7 @@ func (l *SketchLibrariesDetector) IncludeFoldersChanged() bool { // addIncludeFolder add the given folder to the include path. func (l *SketchLibrariesDetector) addIncludeFolder(folder *paths.Path) { + logrus.Tracef("[LD] INCLUDE-PATH: %s", folder.String()) l.includeFolders = append(l.includeFolders, folder) l.cache.Expect(&detectorCacheEntry{AddedIncludePath: folder}) } @@ -219,6 +222,11 @@ func (l *SketchLibrariesDetector) FindIncludes( platformArch string, jobs int, ) error { + logrus.Debug("Finding required libraries for the sketch.") + defer func() { + logrus.Debugf("Library detection completed. Found %d required libraries.", len(l.importedLibraries)) + }() + err := l.findIncludes(ctx, buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch, jobs) if err != nil && l.onlyUpdateCompilationDatabase { l.logger.Info( @@ -272,14 +280,19 @@ func (l *SketchLibrariesDetector) findIncludes( // Pre-run cache entries l.preRunner = runner.New(ctx, jobs) for _, entry := range l.cache.EntriesAhead() { - if entry.Compile != nil && entry.CompileTask != nil { - upToDate, _ := entry.Compile.ObjFileIsUpToDate() + if entry.CompileTask != nil { + upToDate, _ := entry.Compile.ObjFileIsUpToDate(logrus.WithField("runner", "prerun")) if !upToDate { + _ = entry.Compile.PrepareBuildPath() l.preRunner.Enqueue(entry.CompileTask) } } } - defer l.preRunner.Cancel() + defer func() { + if l.preRunner != nil { + l.preRunner.Cancel() + } + }() l.addIncludeFolder(buildCorePath) if buildVariantPath != nil { @@ -290,7 +303,7 @@ func (l *SketchLibrariesDetector) findIncludes( if !l.useCachedLibrariesResolution { sketch := sketch - mergedfile, err := makeSourceFile(sketchBuildPath, sketchBuildPath, paths.New(sketch.MainFile.Base()+".cpp")) + mergedfile, err := l.makeSourceFile(sketchBuildPath, sketchBuildPath, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { return err } @@ -350,17 +363,18 @@ func (l *SketchLibrariesDetector) findIncludes( return nil } -func (l *SketchLibrariesDetector) gccPreprocessTask(sourceFile *sourceFile, buildProperties *properties.Map) *runner.Task { +func (l *SketchLibrariesDetector) gccPreprocessTask(sourceFile sourceFile, buildProperties *properties.Map) *runner.Task { // Libraries may require the "utility" directory to be added to the include // search path, but only for the source code of the library, so we temporary // copy the current search path list and add the library' utility directory // if needed. - includeFolders := l.includeFolders + includeFolders := l.includeFolders.Clone() if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil { includeFolders = append(includeFolders, extraInclude) } - return preprocessor.GCC(sourceFile.SourcePath, paths.NullPath(), includeFolders, buildProperties) + _ = sourceFile.PrepareBuildPath() + return preprocessor.GCC(sourceFile.SourcePath, paths.NullPath(), includeFolders, buildProperties, sourceFile.DepfilePath) } func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( @@ -385,7 +399,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := sourceFile.ObjFileIsUpToDate() + unchanged, err := sourceFile.ObjFileIsUpToDate(logrus.WithField("runner", "main")) if err != nil { return err } @@ -401,11 +415,13 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( var missingIncludeH string if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil { missingIncludeH = *entry.MissingIncludeH + logrus.Tracef("[LD] COMPILE-CACHE: %s", sourceFile.SourcePath) if first && l.logger.VerbosityLevel() == logger.VerbosityVerbose { l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath)) } first = false } else { + logrus.Tracef("[LD] COMPILE: %s", sourceFile.SourcePath) if l.preRunner != nil { if r := l.preRunner.Results(preprocTask); r != nil { preprocResult = r @@ -418,9 +434,8 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( // Stop the pre-runner if l.preRunner != nil { - preRunner := l.preRunner + l.preRunner.Cancel() l.preRunner = nil - go preRunner.Cancel() } // Run the actual preprocessor @@ -446,6 +461,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( } } + logrus.Tracef("[LD] MISSING: %s", missingIncludeH) l.cache.Expect(&detectorCacheEntry{MissingIncludeH: &missingIncludeH}) if missingIncludeH == "" { @@ -493,6 +509,8 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( buildDir *paths.Path, extraIncludePath ...*paths.Path, ) error { + logrus.Tracef("[LD] SCAN: %s (recurse=%v)", folder, recurse) + sourceFileExtensions := []string{} for k := range globals.SourceFilesValidExtensions { sourceFileExtensions = append(sourceFileExtensions, k) @@ -503,7 +521,7 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( } for _, filePath := range filePaths { - sourceFile, err := makeSourceFile(sourceDir, buildDir, filePath, extraIncludePath...) + sourceFile, err := l.makeSourceFile(sourceDir, buildDir, filePath, extraIncludePath...) if err != nil { return err } @@ -513,6 +531,31 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( return nil } +// makeSourceFile create a sourceFile object for the given source file path. +// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder. +func (l *SketchLibrariesDetector) makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (sourceFile, error) { + if len(extraIncludePaths) > 1 { + panic("only one extra include path allowed") + } + var extraIncludePath *paths.Path + if len(extraIncludePaths) > 0 { + extraIncludePath = extraIncludePaths[0] + } + + if sourceFilePath.IsAbs() { + var err error + sourceFilePath, err = sourceRoot.RelTo(sourceFilePath) + if err != nil { + return sourceFile{}, err + } + } + return sourceFile{ + SourcePath: sourceRoot.JoinPath(sourceFilePath), + DepfilePath: buildRoot.Join(fmt.Sprintf("%s.libsdetect.d", sourceFilePath)), + ExtraIncludePath: extraIncludePath, + }, nil +} + func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { if len(l.importedLibraries) == 0 { return nil diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go index d99cb1d862a..088d3de46f6 100644 --- a/internal/arduino/builder/internal/detector/source_file.go +++ b/internal/arduino/builder/internal/detector/source_file.go @@ -19,17 +19,17 @@ import ( "fmt" "slices" - "github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils" + "os" + + "github.com/arduino/arduino-cli/internal/arduino/builder/cpp" "github.com/arduino/go-paths-helper" + "github.com/sirupsen/logrus" ) type sourceFile struct { // SourcePath is the path to the source file SourcePath *paths.Path `json:"source_path"` - // ObjectPath is the path to the object file that will be generated - ObjectPath *paths.Path `json:"object_path"` - // DepfilePath is the path to the dependency file that will be generated DepfilePath *paths.Path `json:"depfile_path"` @@ -41,68 +41,102 @@ type sourceFile struct { } func (f *sourceFile) String() string { - return fmt.Sprintf("SourcePath:%s SourceRoot:%s BuildRoot:%s ExtraInclude:%s", - f.SourcePath, f.ObjectPath, f.DepfilePath, f.ExtraIncludePath) + return fmt.Sprintf("%s -> dep:%s (ExtraInclude:%s)", + f.SourcePath, f.DepfilePath, f.ExtraIncludePath) } // Equals checks if a sourceFile is equal to another. -func (f *sourceFile) Equals(g *sourceFile) bool { +func (f *sourceFile) Equals(g sourceFile) bool { return f.SourcePath.EqualsTo(g.SourcePath) && - f.ObjectPath.EqualsTo(g.ObjectPath) && f.DepfilePath.EqualsTo(g.DepfilePath) && ((f.ExtraIncludePath == nil && g.ExtraIncludePath == nil) || (f.ExtraIncludePath != nil && g.ExtraIncludePath != nil && f.ExtraIncludePath.EqualsTo(g.ExtraIncludePath))) } -// makeSourceFile create a sourceFile object for the given source file path. -// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder. -func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (*sourceFile, error) { - if len(extraIncludePaths) > 1 { - panic("only one extra include path allowed") +// PrepareBuildPath ensures that the directory for the dependency file exists. +func (f *sourceFile) PrepareBuildPath() error { + if f.DepfilePath != nil { + return f.DepfilePath.Parent().MkdirAll() } - var extraIncludePath *paths.Path - if len(extraIncludePaths) > 0 { - extraIncludePath = extraIncludePaths[0] + return nil +} + +// ObjFileIsUpToDate checks if the compile object file is up to date. +func (f *sourceFile) ObjFileIsUpToDate(log *logrus.Entry) (unchanged bool, err error) { + if f.DepfilePath == nil { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: object file or dependency file not provided", f.SourcePath) + return false, nil } - if sourceFilePath.IsAbs() { - var err error - sourceFilePath, err = sourceRoot.RelTo(sourceFilePath) - if err != nil { - return nil, err + sourceFile := f.SourcePath.Clean() + sourceFileStat, err := sourceFile.Stat() + if err != nil { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Could not stat source file: %s", f.SourcePath, err) + return false, err + } + dependencyFile := f.DepfilePath.Clean() + dependencyFileStat, err := dependencyFile.Stat() + if err != nil { + if os.IsNotExist(err) { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Dependency file not found: %v", f.SourcePath, dependencyFile) + return false, nil } + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Could not stat dependency file: %s", f.SourcePath, err) + return false, err } - res := &sourceFile{ - SourcePath: sourceRoot.JoinPath(sourceFilePath), - ObjectPath: buildRoot.Join(sourceFilePath.String() + ".o"), - DepfilePath: buildRoot.Join(sourceFilePath.String() + ".d"), - ExtraIncludePath: extraIncludePath, + if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: %v newer than %v", f.SourcePath, sourceFile, dependencyFile) + return false, nil } - return res, nil -} - -// ObjFileIsUpToDate checks if the compile object file is up to date. -func (f *sourceFile) ObjFileIsUpToDate() (unchanged bool, err error) { - return utils.ObjFileIsUpToDate(f.SourcePath, f.ObjectPath, f.DepfilePath) + deps, err := cpp.ReadDepFile(dependencyFile) + if err != nil { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Could not read dependency file: %s", f.SourcePath, dependencyFile) + return false, err + } + if len(deps.Dependencies) == 0 { + return true, nil + } + if deps.Dependencies[0] != sourceFile.String() { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Depfile is about different source file: %v (expected %v)", f.SourcePath, deps.Dependencies[0], sourceFile) + return false, nil + } + for _, dep := range deps.Dependencies[1:] { + depStat, err := os.Stat(dep) + if os.IsNotExist(err) { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: Not found: %v", f.SourcePath, dep) + return false, nil + } + if err != nil { + logrus.WithError(err).Tracef("[LD] COMPILE-CHECK: REBUILD %v: Failed to read: %v", f.SourcePath, dep) + return false, nil + } + if depStat.ModTime().After(dependencyFileStat.ModTime()) { + log.Tracef("[LD] COMPILE-CHECK: REBUILD %v: %v newer than %v", f.SourcePath, dep, dependencyFile) + return false, nil + } + } + log.Tracef("[LD] COMPILE-CHECK: REUSE %v Up-to-date", f.SourcePath) + return true, nil } // uniqueSourceFileQueue is a queue of source files that does not allow duplicates. -type uniqueSourceFileQueue []*sourceFile +type uniqueSourceFileQueue []sourceFile // Push adds a source file to the queue if it is not already present. -func (queue *uniqueSourceFileQueue) Push(value *sourceFile) { +func (queue *uniqueSourceFileQueue) Push(value sourceFile) { if !queue.Contains(value) { + logrus.Tracef("[LD] QUEUE: Added %s", value.SourcePath) *queue = append(*queue, value) } } // Contains checks if the queue Contains a source file. -func (queue uniqueSourceFileQueue) Contains(target *sourceFile) bool { +func (queue uniqueSourceFileQueue) Contains(target sourceFile) bool { return slices.ContainsFunc(queue, target.Equals) } // Pop removes and returns the first element of the queue. -func (queue *uniqueSourceFileQueue) Pop() *sourceFile { +func (queue *uniqueSourceFileQueue) Pop() sourceFile { old := *queue x := old[0] *queue = old[1:] diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go index dd847c26eda..250cf7ea3ba 100644 --- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go +++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go @@ -45,7 +45,7 @@ func PreprocessSketchWithArduinoPreprocessor( sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp") targetFile := buildPath.Join("preproc", "sketch_merged.cpp") - gccResult := GCC(sourceFile, targetFile, includeFolders, buildProperties).Run(ctx) + gccResult := GCC(sourceFile, targetFile, includeFolders, buildProperties, nil).Run(ctx) verboseOut.Write(gccResult.Stdout) verboseOut.Write(gccResult.Stderr) if gccResult.Error != nil { diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go index f66fbabf5cf..07d90f83a3e 100644 --- a/internal/arduino/builder/internal/preprocessor/ctags.go +++ b/internal/arduino/builder/internal/preprocessor/ctags.go @@ -45,19 +45,20 @@ func PreprocessSketchWithCtags( lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase, verbose bool, ) (*runner.Result, error) { + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + unpreprocessedSourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp.merged") + preprocessedSourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp") + // Create a temporary working directory tmpDir, err := paths.MkTempDir("", "") if err != nil { return nil, err } defer tmpDir.RemoveAll() - ctagsTarget := tmpDir.Join("sketch_merged.cpp") - - stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} // Run GCC preprocessor - sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp") - result := GCC(sourceFile, ctagsTarget, includes, buildProperties).Run(ctx) + ctagsTarget := tmpDir.Join("sketch_merged.cpp") + result := GCC(unpreprocessedSourceFile, ctagsTarget, includes, buildProperties, nil).Run(ctx) stdout.Write(result.Stdout) stderr.Write(result.Stderr) if err := result.Error; err != nil { @@ -69,7 +70,7 @@ func PreprocessSketchWithCtags( fmt.Fprintf(stderr, "%s: %s", i18n.Tr("An error occurred adding prototypes"), i18n.Tr("the compilation database may be incomplete or inaccurate")) - if err := sourceFile.CopyTo(ctagsTarget); err != nil { + if err := unpreprocessedSourceFile.CopyTo(ctagsTarget); err != nil { return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } } @@ -102,7 +103,7 @@ func PreprocessSketchWithCtags( // Add prototypes to the original sketch source var source string - if sourceData, err := sourceFile.ReadFile(); err == nil { + if sourceData, err := unpreprocessedSourceFile.ReadFile(); err == nil { source = string(sourceData) } else { return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err @@ -135,8 +136,15 @@ func PreprocessSketchWithCtags( fmt.Println("#END OF PREPROCESSED SOURCE") } + // Read the existing preprocessed file to check if it's already up-to-date. + oldPreprocessedSource, _ := preprocessedSourceFile.ReadFile() + if bytes.Equal([]byte(preprocessedSource), oldPreprocessedSource) { + // No changes, do nothing + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, nil + } + // Write back arduino-preprocess output to the sourceFile - err = sourceFile.WriteFile([]byte(preprocessedSource)) + err = preprocessedSourceFile.WriteFile([]byte(preprocessedSource)) return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } diff --git a/internal/arduino/builder/internal/preprocessor/gcc.go b/internal/arduino/builder/internal/preprocessor/gcc.go index 31dc2922ab7..6baf2d8391f 100644 --- a/internal/arduino/builder/internal/preprocessor/gcc.go +++ b/internal/arduino/builder/internal/preprocessor/gcc.go @@ -30,6 +30,7 @@ import ( func GCC( sourceFilePath, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map, + depFile *paths.Path, ) *runner.Task { gccBuildProperties := properties.NewMap() gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") @@ -62,6 +63,11 @@ func GCC( // to create a /dev/null.d dependency file, which won't work. args = f.Filter(args, f.NotEquals("-MMD")) + // If a depFile has been specified add the necessary arguments to generate it + if depFile != nil { + args = append(args, "-MMD", "-MF", depFile.String()) + } + // Limit the stderr output to 100 KiB // https://github.com/arduino/arduino-cli/pull/2883 return runner.NewTaskWithLimitedStderr(100*1024, args...) diff --git a/internal/arduino/builder/sketch.go b/internal/arduino/builder/sketch.go index f5196630a9b..cf25afa8abd 100644 --- a/internal/arduino/builder/sketch.go +++ b/internal/arduino/builder/sketch.go @@ -58,9 +58,20 @@ func (b *Builder) prepareSketchBuildPath() error { return err } + // Save the unpreprocessed merged source to a file named sketch.cpp.merged. + destFileUnpreprocessed := b.sketchBuildPath.Join(b.sketch.MainFile.Base() + ".cpp.merged") destFile := b.sketchBuildPath.Join(b.sketch.MainFile.Base() + ".cpp") - if err := destFile.WriteFile([]byte(mergedSource)); err != nil { - return err + oldUnpreprocessedSource, _ := destFileUnpreprocessed.ReadFile() + + // If the merged source is unchanged, skip writing it. + // This avoids unnecessary rebuilds and keeps the build path clean. + if !bytes.Equal(oldUnpreprocessedSource, []byte(mergedSource)) { + if err := destFileUnpreprocessed.WriteFile([]byte(mergedSource)); err != nil { + return err + } + if err := destFile.WriteFile([]byte(mergedSource)); err != nil { + return err + } } if err := b.sketchCopyAdditionalFiles(b.sketchBuildPath, b.sourceOverrides); err != nil { diff --git a/internal/integrationtest/arduino-cli.go b/internal/integrationtest/arduino-cli.go index c990b24c6ae..b8f645cc58c 100644 --- a/internal/integrationtest/arduino-cli.go +++ b/internal/integrationtest/arduino-cli.go @@ -389,22 +389,22 @@ func (cli *ArduinoCLI) run(ctx context.Context, stdoutBuff, stderrBuff io.Writer cli.t.NoError(cliProc.Start()) + if stdoutBuff == nil { + stdoutBuff = io.Discard + } + if stderrBuff == nil { + stderrBuff = io.Discard + } var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() - if stdoutBuff == nil { - stdoutBuff = io.Discard - } if _, err := io.Copy(stdoutBuff, io.TeeReader(stdout, terminalOut)); err != nil { fmt.Fprintln(terminalOut, color.HiBlackString("<<< stdout copy error:"), err) } }() go func() { defer wg.Done() - if stderrBuff == nil { - stderrBuff = io.Discard - } if _, err := io.Copy(stderrBuff, io.TeeReader(stderr, terminalErr)); err != nil { fmt.Fprintln(terminalErr, color.HiBlackString("<<< stderr copy error:"), err) } diff --git a/internal/integrationtest/compile_4/lib_discovery_caching_test.go b/internal/integrationtest/compile_4/lib_discovery_caching_test.go index 9d8bd4b5277..954ab148e83 100644 --- a/internal/integrationtest/compile_4/lib_discovery_caching_test.go +++ b/internal/integrationtest/compile_4/lib_discovery_caching_test.go @@ -28,10 +28,6 @@ func TestLibDiscoveryCache(t *testing.T) { env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) t.Cleanup(env.CleanUp) - // Install Arduino AVR Boards - _, _, err := cli.Run("core", "install", "arduino:avr@1.8.6") - require.NoError(t, err) - // Copy the testdata sketchbook testdata, err := paths.New("testdata", "libraries_discovery_caching").Abs() require.NoError(t, err) @@ -39,72 +35,121 @@ func TestLibDiscoveryCache(t *testing.T) { require.NoError(t, sketchbook.RemoveAll()) require.NoError(t, testdata.CopyDirTo(cli.SketchbookDir())) - buildpath, err := paths.MkTempDir("", "tmpbuildpath") + // Install Arduino AVR Boards + _, _, err = cli.Run("core", "install", "arduino:avr@1.8.6") + require.NoError(t, err) + // Install Ethernet library + _, _, err = cli.Run("lib", "install", "Ethernet") require.NoError(t, err) - t.Cleanup(func() { buildpath.RemoveAll() }) - { + t.Run("RemoveLibWithoutError", func(t *testing.T) { sketchA := sketchbook.Join("SketchA") + buildpath, err := sketchA.Join("build").Abs() + require.NoError(t, err) + t.Cleanup(func() { buildpath.RemoveAll() }) + { + require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` +#include +#include +void setup() {} +void loop() {}`))) outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) require.NoError(t, err) j := requirejson.Parse(t, outjson) - j.MustContain(`{"builder_result":{ - "used_libraries": [ - { "name": "LibA" }, - { "name": "LibB" } - ], - }}`) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["SPI", "Ethernet"]`) } - // Update SketchA - require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` -#include + { + // Update SketchA + require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` +#include +void setup() {} +void loop() {}`))) + // This compile should not include Ethernet + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.NoError(t, err) + j := requirejson.Parse(t, outjson) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["SPI"]`) + j.Query(".compiler_out").MustContain(`"The list of included libraries has been changed... rebuilding all libraries."`) + } + + { + // This compile should not rebuild libs + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.NoError(t, err) + j := requirejson.Parse(t, outjson) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["SPI"]`) + j.Query(".compiler_out").MustNotContain(`"The list of included libraries has changed... rebuilding all libraries."`) + } + }) + + t.Run("RemoveLibWithError", func(t *testing.T) { + sketchA := sketchbook.Join("SketchA") + buildpath, err := sketchA.Join("build").Abs() + require.NoError(t, err) + t.Cleanup(func() { buildpath.RemoveAll() }) + + { + require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` #include void setup() {} -void loop() {libAFunction();} -`))) +void loop() {libAFunction();}`))) + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.NoError(t, err) + j := requirejson.Parse(t, outjson) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["LibA", "LibB"]`) + } { + // Update SketchA + require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` +#include +#include +void setup() {} +void loop() {libAFunction();} + `))) // This compile should FAIL! outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) require.Error(t, err) j := requirejson.Parse(t, outjson) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["LibC", "LibA"]`) j.MustContain(`{ -"builder_result":{ - "used_libraries": [ - { "name": "LibC" }, - { "name": "LibA" } - ], - "diagnostics": [ - { - "severity": "ERROR", - "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" - } - ] -}}`) + "builder_result":{ + "diagnostics": [ + { + "severity": "ERROR", + "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" + } + ] + }}`) j.Query(".compiler_out").MustContain(`"The list of included libraries has been changed... rebuilding all libraries."`) } { + // Compile again the bad sketch + // This compile should FAIL! outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) require.Error(t, err) j := requirejson.Parse(t, outjson) + usedLibs := j.Query("[.builder_result.used_libraries[].name]") + usedLibs.MustEqual(`["LibC", "LibA"]`) j.MustContain(`{ -"builder_result":{ - "used_libraries": [ - { "name": "LibC" }, - { "name": "LibA" } - ], - "diagnostics": [ - { - "severity": "ERROR", - "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" - } - ] -}}`) + "builder_result":{ + "diagnostics": [ + { + "severity": "ERROR", + "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" + } + ] + }}`) j.Query(".compiler_out").MustNotContain(`"The list of included libraries has changed... rebuilding all libraries."`) } - } + }) }