Add managed ad and tracker rule packs

Flowlight/Analysis/BuiltInRulePacks.swift

@@ -0,0 +1,49 @@
+import Foundation
+
+/// Official lists shipped inside the signed app. They are deliberately small and reviewed in 0.15.0; later remote
+/// subscriptions use the same manifest/snapshot boundary but do not change what these built-ins mean.
+enum BuiltInRulePacks {
+    static let all: [RulePackDefinition] = [advertising, tracking, telemetry]
+
+    static let advertising = definition(
+        id: "official.advertising",
+        title: "Advertising",
+        summary: "Known advertising delivery and auction hosts across apps.",
+        caveat: "Connection blocking cannot remove page layout gaps, first-party ads, or overlays drawn by a browser or app.",
+        category: .advertising,
+        hosts: [
+            "adnxs.com", "adsrvr.org", "adservice.google.com", "casalemedia.com", "criteo.com", "criteo.net",
+            "doubleclick.net", "googleadservices.com", "googlesyndication.com", "openx.net", "pubmatic.com",
+            "rubiconproject.com",
+        ])
+
+    static let tracking = definition(
+        id: "official.tracking",
+        title: "Cross-site tracking",
+        summary: "Common analytics, session replay, and cross-site measurement hosts.",
+        caveat: "Some apps use the same providers for feature flags. They may fall back to built-in defaults when a host is unavailable.",
+        category: .tracking,
+        hosts: RuleTemplate.analyticsHosts)
+
+    static let telemetry = definition(
+        id: "official.telemetry",
+        title: "Privacy and telemetry",
+        summary: "Crash reporting, install attribution, and audited Apple advertising and analytics hosts.",
+        caveat: "Crash reports help developers fix failures. Essential Apple account, push, update, certificate, and iCloud traffic stays allowed.",
+        category: .telemetry,
+        hosts: RuleTemplate.crashReportingHosts + RuleTemplate.attributionHosts + RuleTemplate.appleTelemetryHosts)
+
+    private static func definition(id: String, title: String, summary: String, caveat: String,
+                                   category: RulePackEntry.Category, hosts: [String]) -> RulePackDefinition {
+        let canonical = Array(Set(hosts.compactMap(HostCanonicalizer.host))).sorted()
+        let entries = canonical.enumerated().map {
+            RulePackEntry(id: UInt32($0.offset + 1), host: $0.element, match: .suffix, category: category)
+        }
+        let manifest = RulePackManifest(
+            id: id, version: "1", generatedAt: Date(timeIntervalSince1970: 1_792_220_400),
+            entryCount: entries.count, contentSHA256: RulePackValidator.digest(entries),
+            titleKey: title, summaryKey: summary, caveatKey: caveat,
+            source: "Flowlight curated built-in list", license: "GPL-3.0")
+        return RulePackDefinition(manifest: manifest, entries: entries)
+    }
+}

Flowlight/Analysis/RulePackStore.swift

@@ -0,0 +1,198 @@
+import Foundation
+
+/// The app-owned half of managed rule packs: built-in definitions, user activation/scope, exceptions, and the
+/// complete validated snapshot handed to the Network Extension.
+@MainActor
+final class RulePackStore: ObservableObject {
+    enum ApplyStatus: Equatable, Sendable {
+        case off, ready, applying, enforcing, watchingOnly, incompatible(String), failed(String)
+    }
+
+    @Published private(set) var configurations: [RulePackConfiguration] = []
+    @Published private(set) var overrides: [RulePackOverride] = []
+    @Published private(set) var events: [RulePackEventRecord] = []
+    @Published private(set) var deliveryStats = RulePackDeliveryStats(droppedHits: 0, droppedKeys: 0)
+    @Published private(set) var status: ApplyStatus = .off
+    @Published private(set) var version = 0
+
+    let definitions: [RulePackDefinition]
+    var onChange: (RulePackSnapshot?) -> Void = { _ in }
+    private weak var db: TrafficDatabase?
+    private var definitionsValid = true
+
+    init(definitions: [RulePackDefinition] = BuiltInRulePacks.all) {
+        self.definitions = definitions
+        do { try RulePackValidator.validate(definitions) }
+        catch {
+            definitionsValid = false
+            status = .failed("Built-in pack validation failed: \(error)")
+        }
+    }
+
+    var enabledCount: Int { configurations.filter(\.enabled).count }
+    var hasEnabledPacks: Bool { enabledCount > 0 }
+
+    func configuration(for packID: String) -> RulePackConfiguration {
+        configurations.first { $0.packID == packID } ?? RulePackConfiguration(packID: packID)
+    }
+
+    func overrides(for packID: String) -> [RulePackOverride] {
+        overrides.filter { $0.packID == packID }
+    }
+
+    func attach(db: TrafficDatabase) {
+        self.db = db
+        reload()
+    }
+
+    func reload() {
+        guard let db else { return }
+        db.async { [weak self] db in
+            let saved = try db.loadRulePackConfigurations()
+            let exceptions = try db.loadRulePackOverrides()
+            let events = try db.rulePackEvents(limit: 300)
+            let deliveryStats = try db.rulePackDeliveryStats()
+            Task { @MainActor in
+                guard let self else { return }
+                let known = Set(self.definitions.map(\.id))
+                self.configurations = self.definitions.map { definition in
+                    guard var configuration = saved.first(where: { $0.packID == definition.id }) else {
+                        return RulePackConfiguration(packID: definition.id)
+                    }
+                    if configuration.enabled, !configuration.isConsented(to: definition) {
+                        configuration.enabled = false
+                    }
+                    return configuration
+                }
+                self.overrides = exceptions.filter { known.contains($0.packID) }
+                self.events = events.filter { known.contains($0.packID) }
+                self.deliveryStats = deliveryStats
+                self.publish()
+            }
+        }
+    }
+
+    func save(_ configuration: RulePackConfiguration) {
+        guard definitions.contains(where: { $0.id == configuration.packID }) else { return }
+        var cleaned = configuration
+        if cleaned.enabled, let definition = definitions.first(where: { $0.id == cleaned.packID }) {
+            cleaned.consentedVersion = definition.manifest.version
+            cleaned.consentedContentSHA256 = definition.manifest.contentSHA256
+        }
+        cleaned.apps = Array(Set(configuration.apps.map {
+            $0.trimmingCharacters(in: .whitespacesAndNewlines)
+        }.filter { !$0.isEmpty })).sorted()
+        if cleaned.scope == .selectedApps && cleaned.apps.isEmpty { cleaned.enabled = false }
+        persist({ try $0.saveRulePackConfiguration(cleaned) }) { [weak self] in
+            guard let self else { return }
+            if let index = self.configurations.firstIndex(where: { $0.packID == cleaned.packID }) {
+                self.configurations[index] = cleaned
+            } else {
+                self.configurations.append(cleaned)
+            }
+            self.publish()
+        }
+    }
+
+    func setEnabled(_ packID: String, _ enabled: Bool) {
+        var value = configuration(for: packID)
+        value.enabled = enabled
+        save(value)
+    }
+
+    func addOverride(_ override: RulePackOverride) {
+        guard definitions.contains(where: { $0.id == override.packID }),
+              let host = HostCanonicalizer.host(override.host) else { return }
+        var cleaned = override
+        cleaned.host = host
+        cleaned.app = override.app.trimmingCharacters(in: .whitespacesAndNewlines)
+        persist({ try $0.saveRulePackOverride(cleaned) }) { [weak self] in
+            guard let self else { return }
+            self.overrides.removeAll { $0.id == cleaned.id }
+            self.overrides.append(cleaned)
+            self.publish()
+        }
+    }
+
+    func deleteOverride(_ override: RulePackOverride) {
+        persist({ try $0.deleteRulePackOverride(override.id) }) { [weak self] in
+            self?.overrides.removeAll { $0.id == override.id }
+            self?.publish()
+        }
+    }
+
+    func updateStatus(_ newStatus: ApplyStatus) { status = newStatus }
+
+    func markApplying() {
+        guard definitionsValid else { return }
+        status = .applying
+    }
+
+    func record(_ batch: RulePackEventBatch, completion: @escaping (Bool) -> Void) {
+        guard let db else { completion(false); return }
+        db.queue.async { [weak self, weak db] in
+            guard let db else { completion(false); return }
+            do {
+                _ = try db.recordRulePackEventBatch(batch)
+                let events = try db.rulePackEvents(limit: 300)
+                let deliveryStats = try db.rulePackDeliveryStats()
+                Task { @MainActor in
+                    self?.events = events
+                    self?.deliveryStats = deliveryStats
+                    self?.version += 1
+                    completion(true)
+                }
+            } catch {
+                appLog.error("rule pack batch failed: \(String(describing: error), privacy: .public)")
+                Task { @MainActor in completion(false) }
+            }
+        }
+    }
+
+    func events(for packID: String) -> [RulePackEventRecord] { events.filter { $0.packID == packID } }
+
+    func clearActivity() {
+        events = []
+        deliveryStats = RulePackDeliveryStats(droppedHits: 0, droppedKeys: 0)
+        version += 1
+    }
+
+    var snapshot: RulePackSnapshot? {
+        let definitionsByID = Dictionary(uniqueKeysWithValues: definitions.map { ($0.id, $0) })
+        let active = configurations.filter { configuration in
+            configuration.enabled
+                && definitionsByID[configuration.packID].map(configuration.isConsented(to:)) == true
+        }
+        guard !active.isEmpty else { return nil }
+        let packIDs = Set(active.map(\.packID))
+        let included = definitions.filter { packIDs.contains($0.id) }
+        let includedOverrides = overrides.filter { packIDs.contains($0.packID) }
+        return RulePackSnapshot.make(definitions: included, configurations: active, overrides: includedOverrides)
+    }
+
+    private func persist(_ write: @escaping (TrafficDatabase) throws -> Void,
+                         then commit: @escaping @MainActor () -> Void) {
+        guard let db else {
+            status = .failed("Rule pack storage is unavailable.")
+            return
+        }
+        db.queue.async { [weak self, weak db] in
+            guard let db else { return }
+            do {
+                try write(db)
+                Task { @MainActor in commit() }
+            } catch {
+                Task { @MainActor in
+                    self?.status = .failed("Could not save the rule pack change: \(error.localizedDescription)")
+                }
+            }
+        }
+    }
+
+    private func publish() {
+        version += 1
+        guard definitionsValid else { return }
+        status = hasEnabledPacks ? .ready : .off
+        onChange(snapshot)
+    }
+}

Flowlight/App/TrafficMonitor.swift

@@ -54,6 +54,8 @@ final class TrafficMonitor: ObservableObject {
     let exporter = ExportController()
     /// Everything Flowlight has been told to block or allow, and what those rules have done.
     let rules = RuleStore()
+    /// Official managed destination lists, kept separate from human-authored rules and disabled by default.
+    let rulePacks = RulePackStore()
     /// Which tools each agent may use, which is a different question from which hosts it may reach.
     let guardrails = GuardrailStore()
     /// Advisory inspection plugins: official built-ins first, installed third-party packs later.
@@ -153,6 +155,13 @@ final class TrafficMonitor: ObservableObject {
         Task { await localRisk.recover() }
         rules.onChange = { [weak self] set in self?.source?.setRules(set) }
         rules.attach(db: db)
+        rulePacks.onChange = { [weak self] snapshot in
+            guard let self else { return }
+            if self.source is ExtensionTrafficSource { self.rulePacks.markApplying() }
+            else { self.rulePacks.updateStatus(self.rulePacks.hasEnabledPacks ? .watchingOnly : .off) }
+            self.source?.setRulePacks(snapshot)
+        }
+        rulePacks.attach(db: db)
         // A rule that names a path is carried out by the proxy, which reads its list fresh on every connection.
         inspection.requestRules = { [live = rules.live] in live.requestRules() }
         inspection.onRuleRefusal = { [weak self] event in
@@ -365,6 +374,12 @@ final class TrafficMonitor: ObservableObject {
             extensionSource.onRuleDecisions = { [weak self] events in
                 Task { @MainActor in self?.rules.record(events) }
             }
+            extensionSource.onRulePackEvents = { [weak self] batch, completion in
+                self?.recordRulePackEvents(batch, completion: completion) ?? completion(false)
+            }
+            extensionSource.onRulePackStatus = { [weak self] result in
+                Task { @MainActor in self?.updateRulePackStatus(result) }
+            }
             extensionSource.onRepairRequested = { [weak self] in
                 Task { @MainActor in self?.repairExtensionVersion() }
             }
@@ -382,6 +397,35 @@ final class TrafficMonitor: ObservableObject {
         })
         pushEnforcement()
         newSource.setRules(rules.ruleSet)
+        newSource.setRulePacks(rulePacks.snapshot)
+        if rulePacks.hasEnabledPacks, !(newSource is ExtensionTrafficSource) {
+            rulePacks.updateStatus(.watchingOnly)
+        }
+    }
+
+    private func updateRulePackStatus(_ result: RulePackApplyResult) {
+        switch result.status {
+        case .accepted:
+            rulePacks.updateStatus(.applying)
+        case .committed:
+            rulePacks.updateStatus(.enforcing)
+        case .cleared:
+            rulePacks.updateStatus(.off)
+        case .incompatible:
+            rulePacks.updateStatus(.incompatible(result.detail))
+        case .invalid, .stale:
+            rulePacks.updateStatus(.failed(result.detail))
+        }
+    }
+
+    nonisolated private func recordRulePackEvents(_ batch: RulePackEventBatch,
+                                                   completion: @escaping (Bool) -> Void) {
+        Task { @MainActor in
+            self.rulePacks.record(batch) { [weak self] committed in
+                if committed { self?.dataVersion += 1 }
+                completion(committed)
+            }
+        }
     }
 
     /// The extension source has been refused often enough that plain redialling isn't the answer. Ask macOS which
@@ -610,6 +654,9 @@ final class TrafficMonitor: ObservableObject {
             for hour in completed.completedHours { try engine.hourCompleted(hour) }
             for day in completed.completedDays { try engine.dayCompleted(day) }
             try engine.evaluateIdleTraffic()
+            let eventCutoff = Date().addingTimeInterval(-max(1, retentionHours) * 3600)
+            try db.pruneRuleEvents(before: eventCutoff)
+            try db.pruneRulePackEvents(before: eventCutoff)
             let coverage = try db.coverage(since: Date().addingTimeInterval(-3600))
             Task { @MainActor in
                 self?.coverage = coverage
@@ -663,6 +710,7 @@ final class TrafficMonitor: ObservableObject {
                 self?.perSecond.removeAll()
                 self?.sessionIn = 0
                 self?.sessionOut = 0
+                self?.rulePacks.clearActivity()
                 self?.dataVersion += 1
                 self?.refreshAlertCount()
             }