feat: Twitch VOD support

.coderabbit.yaml

@@ -0,0 +1,8 @@
+reviews:
+  auto_review:
+    enabled: true
+    base_branches:
+      - staging
+  path_filters:
+    - "!stream_pb2.py"
+    - "!stream_pb2_grpc.py"

README.md

@@ -29,6 +29,9 @@ Why not get some use out of it? Archivists everywhere, rejoice!
 | `-batch`       | Time betwen URL checks (seconds): increase for rate limiting | `5`               |
 | `-subfolder`   | Add streams to a subfolder with the channel name             | `false`           |
 | `-log-level`   | Set logging level (debug, info, warn, error, etc)            | `info`            |
+| `-data`        | Directory for persistent data (VOD tracking database)        | `/app/data`       |
+| `-vod-out`     | Output location for VOD downloads (defaults to `-out`)       | Same as `-out`    |
+| `-vod-move`    | Move location for completed VOD downloads (defaults to `-move`) | Same as `-move` |
 
 ## Install
 
@@ -113,6 +116,40 @@ Basic YAML format. See `config/config.yaml.example` for a couple of test sites.
       quality: best
 ```
 
+## VOD Downloads (Twitch)
+
+StreamDL can download past broadcasts (VODs) from Twitch. Enable per-channel with the `vod` option:
+
+```yaml
+- site: twitch.tv
+  channels:
+  - name: day9tv
+    quality: best
+    vod: true
+    vod_limit: 5  # Check the 5 most recent VODs (default: 10)
+```
+
+**How it works:**
+- On each tick, StreamDL checks for new VODs using yt-dlp
+- Downloaded VODs are tracked in a SQLite database (default `/app/data/streamdl.db`, configurable via `-data`) to avoid re-downloading
+- In-progress downloads are tracked so interrupted downloads are retried after a stale threshold
+- VOD files are named: `{user}_vod_{id}_{title}.mp4`
+- Stream copy is used by default (no re-encoding) for fast downloads
+
+**Docker volume:** Mount the data directory to persist the VOD tracking database across container restarts. If using the default `-data` path:
+
+```yaml
+volumes:
+  - ./data:/app/data
+```
+
+**Separate output directories:** Use `-vod-out` and `-vod-move` to send VODs to a different location than live streams. If not set, VODs use the same `-out` and `-move` directories.
+
+**Notes:**
+- `vod: true` and live streaming are mutually exclusive per channel entry
+- To download both live streams and VODs, add the same channel twice with different modes
+- Currently supported for Twitch only
+
 ## Environment Variables
 
 StreamDL supports configuration through environment variables for certain system-level settings.

config.go

@@ -8,6 +8,8 @@ type Config struct {
 
 // Streamer definition
 type Streamer struct {
-	User    string `yaml:"name"`
-	Quality string `yaml:"quality"`
+	User     string `yaml:"name"`
+	Quality  string `yaml:"quality"`
+	VOD      bool   `yaml:"vod"`
+	VODLimit int    `yaml:"vod_limit"`
 }

config/config.yml.example

@@ -4,6 +4,8 @@
     quality: best
   - name: day9tv
     quality: worst
+    vod: true       # Download VODs instead of live streams
+    vod_limit: 5    # Check the 5 most recent VODs (default: 10)
 - site: mixer.com
   channels:
   - name: ninja

config_reader_test.go

@@ -36,6 +36,45 @@ func TestReadConfig_MissingFile_Fatal(t *testing.T) {
 	}
 }
 
+func TestParseConfig_VODFields(t *testing.T) {
+	yamlData := []byte(`
+- site: twitch.tv
+  channels:
+  - name: testuser
+    quality: best
+    vod: true
+    vod_limit: 5
+- site: twitch.tv
+  channels:
+  - name: liveuser
+    quality: best
+`)
+	config, err := parseConfig(yamlData)
+	if err != nil {
+		t.Fatalf("Failed to parse config: %v", err)
+	}
+
+	if len(config) != 2 {
+		t.Fatalf("Expected 2 site configs, got %d", len(config))
+	}
+
+	vodStreamer := config[0].Streamers[0]
+	if !vodStreamer.VOD {
+		t.Error("Expected VOD to be true")
+	}
+	if vodStreamer.VODLimit != 5 {
+		t.Errorf("Expected VODLimit 5, got %d", vodStreamer.VODLimit)
+	}
+
+	liveStreamer := config[1].Streamers[0]
+	if liveStreamer.VOD {
+		t.Error("Expected VOD to default to false")
+	}
+	if liveStreamer.VODLimit != 0 {
+		t.Errorf("Expected VODLimit 0 (default), got %d", liveStreamer.VODLimit)
+	}
+}
+
 func TestParseConfig_MalformedYAML(t *testing.T) {
 	dir := t.TempDir()
 	cfg := filepath.Join(dir, "bad.yml")

download_stream.go

@@ -417,3 +417,161 @@ func redactBetween(s, start, end string) string {
 	j += idx
 	return s[:idx+len(start)] + "<redacted>" + s[j:]
 }
+
+// sanitizeFilename removes or replaces characters that are unsafe in filenames.
+func sanitizeFilename(name string) string {
+	replacer := strings.NewReplacer(
+		"/", "_", "\\", "_", ":", "_", "*", "_",
+		"?", "_", "\"", "_", "<", "_", ">", "_",
+		"|", "_", "\n", "_", "\r", "_",
+	)
+	sanitized := replacer.Replace(name)
+	// Collapse multiple underscores
+	for strings.Contains(sanitized, "__") {
+		sanitized = strings.ReplaceAll(sanitized, "__", "_")
+	}
+	// Trim to reasonable length
+	if len(sanitized) > 100 {
+		sanitized = sanitized[:100]
+	}
+	return strings.Trim(sanitized, "_. ")
+}
+
+// downloadVOD downloads a single VOD and updates its status in the database.
+// The url parameter is a resolved stream URL (from GetStream via Streamlink/yt-dlp).
+func downloadVOD(user string, vod VodResult, url string, outLoc string, moveLoc string, subfolder bool, vodDB *VodDB, control <-chan bool, response chan<- bool) {
+	sanitizedTitle := sanitizeFilename(vod.Title)
+	fileBase := user + "_vod_" + vod.ID
+	if sanitizedTitle != "" {
+		fileBase += "_" + sanitizedTitle
+	}
+
+	naturalFinish := make(chan error, 1)
+	sigint := make(chan bool)
+
+	// Always ensure base directories have correct permissions first
+	if err := createDirWithUmask(outLoc); err != nil {
+		log.Errorf("Failed to create output directory %s: %v", outLoc, err)
+		response <- true
+		return
+	}
+	if err := createDirWithUmask(moveLoc); err != nil {
+		log.Errorf("Failed to create move directory %s: %v", moveLoc, err)
+		response <- true
+		return
+	}
+
+	if subfolder {
+		outLoc = filepath.Join(outLoc, user)
+		if err := createDirWithUmask(outLoc); err != nil {
+			log.Errorf("Failed to create output subfolder %s: %v", outLoc, err)
+			response <- true
+			return
+		}
+		moveLoc = filepath.Join(moveLoc, user)
+		if err := createDirWithUmask(moveLoc); err != nil {
+			log.Errorf("Failed to create move subfolder %s: %v", moveLoc, err)
+			response <- true
+			return
+		}
+	}
+
+	outPath := filepath.Join(outLoc, fileBase+".mp4")
+	newPath := filepath.Join(moveLoc, fileBase+".mp4")
+	log.Infof("Starting VOD download for %s: %s", user, vod.Title)
+
+	// Single control listener
+	go func() {
+		for {
+			_, more := <-control
+			if !more {
+				sigint <- true
+				return
+			}
+		}
+	}()
+
+	buf := &bytes.Buffer{}
+	cmd := fluentffmpeg.
+		NewCommand("").
+		InputPath(url).
+		OutputFormat("mp4").
+		OutputPath(outPath).
+		OutputLogs(buf).
+		Build()
+
+	// Prefer stream copy for VODs to avoid re-encoding
+	outIdx := indexOf(cmd.Args, outPath)
+	copyArgs := []string{"-c:v", "copy", "-c:a", "copy", "-movflags", "+faststart"}
+	if outIdx == -1 {
+		cmd.Args = append(cmd.Args, copyArgs...)
+	} else {
+		newArgs := make([]string, 0, len(cmd.Args)+len(copyArgs))
+		newArgs = append(newArgs, cmd.Args[:outIdx]...)
+		newArgs = append(newArgs, copyArgs...)
+		newArgs = append(newArgs, cmd.Args[outIdx:]...)
+		cmd.Args = newArgs
+	}
+
+	if indexOf(cmd.Args, "-y") == -1 {
+		cmd.Args = insertAfterBinary(cmd.Args, []string{"-y"})
+	}
+	log.Debugf("FFmpeg VOD args (sanitized): %s", sanitizeArgs(cmd.Args))
+
+	if err := cmd.Start(); err != nil {
+		log.Errorf("Failed to start FFmpeg for VOD %s: %v", vod.ID, err)
+		if vodDB != nil {
+			vodDB.MarkVODFailed(vod.ID)
+		}
+		response <- true
+		return
+	}
+
+	go func() {
+		naturalFinish <- cmd.Wait()
+	}()
+
+	select {
+	case <-sigint:
+		log.Tracef("Sending SIGINT to VOD %s process", vod.ID)
+		if err := cmd.Process.Signal(syscall.SIGINT); err != nil {
+			log.Errorf("Failed to send SIGINT to VOD %s: %v", vod.ID, err)
+		}
+		cmd.Process.Wait()
+		cmd.Wait()
+		// Interrupted — leave as 'downloading'; stale threshold will handle retry
+		response <- true
+		return
+	case err := <-naturalFinish:
+		if err != nil {
+			log.Warnf("FFmpeg failed for VOD %s: %v", vod.ID, err)
+			ffLog := tailString(buf.String(), 50)
+			if ffLog != "" {
+				log.Warnf("FFmpeg log tail for VOD %s:\n%s", vod.ID, sanitizeLog(ffLog))
+			}
+			if vodDB != nil {
+				vodDB.MarkVODFailed(vod.ID)
+			}
+			response <- true
+			return
+		}
+
+		log.Debugf("VOD %s download complete", vod.ID)
+		if err := moveFile(outPath, newPath); err != nil {
+			log.Errorf("Failed to move VOD file: %v", err)
+			if vodDB != nil {
+				vodDB.MarkVODFailed(vod.ID)
+			}
+		} else {
+			log.Debugf("Moved VOD to %v", newPath)
+			if vodDB != nil {
+				if err := vodDB.MarkVODCompleted(vod.ID); err != nil {
+					log.Errorf("Failed to mark VOD %s as completed: %v", vod.ID, err)
+				}
+			}
+		}
+
+		response <- true
+		return
+	}
+}

entrypoint_client.sh

@@ -12,8 +12,8 @@ if ! getent passwd "${PUID}" >/dev/null 2>&1; then
   adduser -D -u "${PUID}" -G streamdl streamdl
 fi
 
-# Ensure download directories exist and are writable by the runtime user
-mkdir -p /app/dl /app/out
-chown "${PUID}:${PGID}" /app/dl /app/out 2>/dev/null || \
-  echo "Could not change ownership on /app/dl or /app/out"
+# Ensure download and data directories exist and are writable by the runtime user
+mkdir -p /app/dl /app/out /app/data
+chown "${PUID}:${PGID}" /app/dl /app/out /app/data 2>/dev/null || \
+  echo "Could not change ownership on /app/dl, /app/out, or /app/data"
 exec su-exec "${PUID}":"${PGID}" /app/streamdl_client_entrypoint.sh "$@"

go.mod

@@ -14,24 +14,32 @@ require (
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/stretchr/testify v1.11.1 // indirect
 	go.opentelemetry.io/otel v1.41.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
 	golang.org/x/crypto v0.47.0 // indirect
 	golang.org/x/net v0.49.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/text v0.33.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	modernc.org/libc v1.70.0 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/sqlite v1.48.2 // indirect
 )

go.sum

@@ -4,6 +4,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -48,6 +50,8 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQ
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/modfy/fluent-ffmpeg v0.1.0 h1:9T191rhSK6KfoDo9Y/+0Tph3khrudvLQEEi05O+ijHA=
 github.com/modfy/fluent-ffmpeg v0.1.0/go.mod h1:GauXGqGYAmYFupCWG8n1eyuLZMKmLxGTGvszYkJ0Oyo=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -64,6 +68,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
@@ -118,6 +124,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
 golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
 golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -157,3 +165,11 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw=
+modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/sqlite v1.48.2 h1:5CnW4uP8joZtA0LedVqLbZV5GD7F/0x91AXeSyjoh5c=
+modernc.org/sqlite v1.48.2/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=