diff --git a/.claude/rules/documents-frontmatter.md b/.claude/rules/documents-frontmatter.md index 93177154b..f10c3b189 100644 --- a/.claude/rules/documents-frontmatter.md +++ b/.claude/rules/documents-frontmatter.md @@ -21,7 +21,6 @@ globs: documents/**/*.md | 字段 | 类型 | 说明 | |------|------|------| -| `description` | string | 一句话摘要，缺失时由 `documents/hooks/meta.py` 自动生成，但建议手动填写以控制质量 | | `description` | string | 一句话摘要，建议手动填写以控制质量 | | `tags` | list[string] | 分类标签，必须来自下方 VALID_TAGS 集合 | diff --git a/.env.example b/.env.example deleted file mode 100644 index 47473f95d..000000000 --- a/.env.example +++ /dev/null @@ -1,6 +0,0 @@ -# Claude API 配置（用于 cppref_card_generator.py 和 translate.py） -# 不要将此文件提交到 Git — 复制为 .env 并填入真实值 - -ANTHROPIC_AUTH_TOKEN=your-anthropic-api-key-here -ANTHROPIC_BASE_URL=https://api.anthropic.com -ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-20250514 diff --git a/.gitignore b/.gitignore index 44e0eccc7..ee5b8bac8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .claude/prompts/vol5-code-verification.md .claude/chapter-projects-outline.md .claude/templates/ +.claude/tools/ # Build directories build/ @@ -28,8 +29,5 @@ __pycache__/ # cppreference crawler cache (contains API keys in env) scripts/cppref_cache/ -# Environment variables -.env - # Ignore Any Caches brought from local .cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af0c87992..5cc69fd51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,18 +10,9 @@ repos: files: '^documents/.*\.md$' exclude: '^documents/.*\bindex\.md$' - # Frontmatter validation (local hook) + # Local hooks - repo: local hooks: - - id: validate-frontmatter - name: Validate article frontmatter - entry: scripts/validate_frontmatter.py - language: python - additional_dependencies: ['pyyaml'] - files: '^documents/core-embedded-cpp/.*\.md$' - exclude: '(index\.md|tags\.md)$' - pass_filenames: false - - id: clang-format name: clang-format C/C++ sources entry: clang-format -i @@ -35,13 +26,6 @@ repos: always_run: true pass_filenames: false - - id: check-bold-rendering - name: Check bold rendering (**) - entry: pnpm exec tsx scripts/check_bold_rendering.ts - language: system - files: '^documents/.*\.md$' - pass_filenames: false - # Check for added large files - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 diff --git a/documents/en/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md b/documents/en/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md index 98698cb95..c23c62f4e 100644 --- a/documents/en/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md +++ b/documents/en/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md @@ -11,7 +11,7 @@ prerequisites: - 'Chapter 4: std::optional' reading_time_minutes: 11 related: -- if/switch 初始化器 +- if/switch initializers tags: - host - cpp-modern @@ -19,311 +19,371 @@ tags: title: 'Structured Binding: Unpacking Multiple Values in One Line' translation: source: documents/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md - source_hash: 97fac40ee9565ca01e3a1eb7cae4fe21b39f845573e6ecd7db9f3c865d28793e - translated_at: '2026-06-16T03:57:49.673279+00:00' - engine: anthropic - token_count: 2103 + source_hash: 7d6b4bfb92245b45aca6741e936ad295899a1c75d9b52333111a9b1f51fe3dec + translated_at: '2026-07-05T04:06:39.000000+00:00' + engine: manual + token_count: 2200 --- # Structured Binding: Unpacking Multiple Values in One Line -When writing code, I often encounter an awkward scenario: a function returns multiple values, and I have to unpack them one by one and assign them to variables. Using `std::pair` means writing `.first` and `.second`, and using `std::tuple` means writing `std::get<0>` and `std::get<1>`—either the semantics are unclear, or the syntax is ugly. C++11 introduced `std::tie` to alleviate this problem, but honestly, the syntax isn't elegant either: you have to declare all the variables first, and then use `std::tie` to stuff values into them. Is there a feature that feels as good as Python's multi-value unpacking? Yes, there is, folks! +When I'm writing code, I keep bumping into an awkward scenario: a function returns multiple values, and you have to unpack them one by one into variables. With `pair` you write `result.first`, `result.second`; with `tuple` you write `std::get<0>(t)`—either the semantics are unclear or the syntax is ugly. C++11 brought `std::tie` to ease this, but honestly that syntax isn't elegant either: you declare all the variables first, then use `tie` to stuff values in. Isn't there something as satisfying as Python's `a, b = func()`? Yes, there is, folks. -C++17 finally gave us a real answer—Structured Binding. One line of code unpacks `std::pair`, `std::tuple`, arrays, and structs directly into named variables. The semantics are clear, and the overhead is zero. - -> TL;DR: **Structured binding allows you to "unpack" compound types into multiple named variables, while the compiler handles everything behind the scenes.** +C++17 finally gave a real answer—Structured Binding. One line unpacks `pair`, `tuple`, arrays, and structs into named variables. Clear semantics, zero overhead. ------ -## Step 1 — Binding pair and tuple +## Starting with pair and tuple -### pair: The Most Common Multi-Return Value +### pair: the most common multi-return -`std::pair` is the most common way to "pack two values" in the standard library. `std::map::insert` returns a `std::pair`, and `std::map::find` returns an iterator to a `std::pair`. Before structured binding, we had to write this: +`std::pair` is the most common "pack two values" type in the standard library. `std::map::insert` returns a `pair`, and `std::map::find` returns a `pair&`. Before structured binding, you had to write: ```cpp -auto result = my_map.insert(...); +auto result = m.insert({1, "one"}); if (result.second) { - // ... + std::cout << "Inserted: " << result.first->second << '\n'; } ``` -What does `result.second` mean? Without checking the documentation, you have no idea. Structured binding writes the semantics directly into the variable names: +What does `result.second` mean? Without checking docs you have no idea. Structured binding writes the semantics straight into the variable names: ```cpp -auto [iter, success] = my_map.insert(...); -if (success) { - // ... +auto [it, inserted] = m.insert({1, "one"}); +if (inserted) { + std::cout << "Inserted: " << it->second << '\n'; } ``` -It is incredibly elegant when iterating over a map in a range-based for loop. Previously, you would write `it->first` and `it->second`; now, you can write `key` and `value` directly: +It's downright elegant when iterating a map in a range-based for loop. You used to write `it->first` and `it->second`; now it's just `[key, value]`: ```cpp -for (const auto& [key, value] : my_map) { - std::cout << key << ": " << value << '\n'; +std::map sensor_names = { + {1, "Temperature"}, + {2, "Humidity"}, + {3, "Pressure"} +}; + +for (const auto& [id, name] : sensor_names) { + std::cout << "Sensor " << +id << ": " << name << '\n'; } ``` -> Why write `'\n'` instead of `std::endl`? Because `std::endl` outputs a newline character **and** flushes the buffer, which can significantly slow down I/O performance. `'\n'` only outputs a newline. +One detail: the loop body writes `+id`, not `id`. Why? Because `uint8_t`'s `operator<<` treats it as a char, while `+` performs integral promotion, forcing it to `int` before printing. Don't take my word for it—running it is the clearest proof (GCC 16.1.1, `-O2`): + +```text +without + (raw uint8_t): A +with + (promoted) : 65 +``` -### tuple: Cases with More Than Two Values +Same `id = 65`: without `+` you get the char `A`; with `+` you get the number. -When a function needs to return three or more values, `std::tuple` is the natural choice. The syntax for structured binding is exactly the same as for `pair`: +### tuple: more than two values + +When a function needs to return three or more values, `std::tuple` is the natural choice. The structured-binding syntax is exactly the same as for `pair`: ```cpp -std::tuple get_coords() { - return {10, 20, 30}; +std::tuple query_database(int id) { + return {id, "sensor_" + std::to_string(id), 23.5}; } -auto [x, y, z] = get_coords(); +auto [record_id, name, value] = query_database(42); ``` -### Comparison with std::tie +### Compared to std::tie -C++11's `std::tie` can do something similar, but the experience is much worse. It requires declaring all variables first, then using `std::tie` to assign values to them: +C++11's `std::tie` can do something similar, but the ergonomics are noticeably worse. It makes you declare all variables first, then assign through `tie`: ```cpp -int x, y, z; -std::tie(x, y, z) = get_coords(); +int record_id; +std::string name; +double value; +std::tie(record_id, name, value) = query_database(42); ``` -The comparison is obvious: structured binding combines variable declaration and unpacking in one step, whereas `std::tie` requires two steps. Although `std::tie` uses references internally (meaning it can handle tuples containing non-copyable types like `std::unique_ptr` because reference binding doesn't involve copying), structured binding offers cleaner syntax and supports multiple semantics: by value, by reference, and by forwarding reference. +The comparison is obvious: structured binding does declaration and unpacking in one step, while `std::tie` needs two. `std::tie` does use references internally, so it can handle tuples with non-copyable types (like `std::unique_ptr`)—reference binding doesn't copy. But structured binding has cleaner syntax and supports multiple semantics: by value, by reference, by forwarding reference. ------ -## Step 2 — Binding Native Arrays and Structs +## Native arrays and structs -### Native Arrays +### Native arrays -Fixed-size native arrays can also be unpacked directly. This is very convenient when processing data in a fixed format: +Fixed-size native arrays unpack directly too. Handy when processing fixed-format data: ```cpp -int arr[3] = {1, 2, 3}; -auto [a, b, c] = arr; +int rgb[3] = {255, 128, 0}; +auto [r, g, b] = rgb; ``` -Each row of a two-dimensional array can also be unpacked in a loop: +Each row of a 2D array can be unpacked in a loop: ```cpp -int matrix[2][3] = {{1, 2, 3}, {4, 5, 6}}; - -for (auto [x, y, z] : matrix) { - std::cout << x << ", " << y << ", " << z << '\n'; +int matrix[2][3] = { + {1, 2, 3}, {4, 5, 6} +}; +for (auto& row : matrix) { + auto [a, b, c] = row; + std::cout << a << ' ' << b << ' ' << c << '\n'; } ``` -Note that structured binding only supports direct unpacking of one-dimensional arrays. You cannot write `auto [x, y, z] = matrix`, because `matrix[0]` is essentially `int[3]`, whose size is 2, not 6. +Note that structured binding only supports direct unpacking of one-dimensional arrays. You can't write `auto [a, b, c, d, e, f] = matrix`, because `matrix` is essentially `int[2][3]`—its size is 2, not 6. -### Structs and Classes +### Structs and classes -If all non-static data members of a struct are `public`, it can be unpacked directly by structured binding. The compiler binds them in declaration order: +If every non-static data member of a struct is `public`, the struct can be unpacked directly. The compiler binds members in declaration order: ```cpp -struct Point { - double x; - double y; +struct SensorReading { + uint8_t sensor_id; + float value; + uint32_t timestamp; + bool is_valid; }; -Point get_point() { - return {3.5, 4.5}; -} - -auto [x, y] = get_point(); +SensorReading reading{5, 23.5f, 1234567890, true}; +auto [id, val, ts, valid] = reading; ``` -This is arguably the most intuitive usage of structured binding. You don't even need to understand template metaprogramming; as long as the struct members are public, you can use it. +No template metaprogramming needed—as long as the members are public, it just works. This is arguably the most intuitive use of structured binding. -Structured binding requires data members to be bound in declaration order and fully supports bit fields. If the struct contains `const` members, behavior needs attention: the bound "anonymous variable" might be `const`-qualified, but `mutable` members are not restricted by this and can still be modified. +Structured binding requires members to be bound in declaration order, and it fully supports bit fields. If the struct has `mutable` members, watch out: the bound "anonymous variable" may be `const`-qualified, but `mutable` members aren't affected and stay modifiable. ------ -## Step 3 — Understanding the Three Semantics of Binding +## The three binding semantics -Structured binding does not always copy. In fact, the modifier before `auto` determines the type of the underlying anonymous variable: +Structured binding doesn't always copy. The modifier in front of `auto` decides the type of the underlying anonymous variable: -- **`auto`** — Copy by value. The bound variables refer to this copy. -- **`auto&`** — Bind to an lvalue reference. Allows modification of the original object. -- **`const auto&`** — Bind to a const lvalue reference. Read-only access, no copy. -- **`auto&&`** — Forwarding reference. Can bind to both lvalues and rvalues. +- **`auto [...]`**—copy by value. The bound names refer to this copy. +- **`auto& [...]`**—binds to an lvalue reference. You can modify the original. +- **`const auto& [...]`**—binds to a const lvalue reference. Read-only, no copy. +- **`auto&& [...]`**—forwarding reference. Binds to both lvalues and rvalues. -Here is an example to distinguish them: +One example to tell them apart: ```cpp -std::tuple get_tuple() { - return {42, "hello"}; -} +std::pair range{1, 10}; -// 1. Copy: x and y are copies of the tuple elements -auto [x1, y1] = get_tuple(); +// Copy: r1, r2 refer to an anonymous copy, don't affect range +auto [r1, r2] = range; -// 2. Reference: x and y refer to the original object's elements -auto& [x2, y2] = t; +// Reference: operate on the original directly +auto& [r3, r4] = range; +r3 = 5; // range.first becomes 5 +``` -// 3. Const reference: read-only access -const auto& [x3, y3] = get_tuple(); +Run it and you can see `auto&` mutates the original, `auto` mutates a copy: -// 4. Forwarding reference: deduces based on the value category of the initializer -auto&& [x4, y4] = get_tuple(); // Binds to rvalue reference +```text +range.first after auto& mutation: 5 +r1 (copy, unaffected) : 1 ``` -The underlying mechanism is this: the compiler first declares an anonymous variable (type determined by `auto`/`auto&`/`const auto&`/`auto&&`) and initializes it with the expression on the right. Then, each bound variable is a reference to a member of this anonymous variable (or, in the case of by-value, a reference to a member of the copy). +The underlying mechanism: the compiler first declares an anonymous variable (type decided by `auto`/`auto&`/`const auto&`/`auto&&`) and initializes it with the right-hand side. Then each bound variable is a reference to a member of that anonymous variable (or, for the by-value case, a reference to a member of the copy). ```cpp -// Compiler roughly transforms this: -auto [x, y] = get_tuple(); - -// Into this: -auto __anonymous = get_tuple(); -using E = std::remove_reference_t; -auto& x = std::get<0>(__anonymous); -auto& y = std::get<1>(__anonymous); +// auto [x, y] = get_point(); is roughly equivalent to: +auto __anonymous = get_point(); +auto& x = __anonymous.first; // refers to the anonymous variable's member +auto& y = __anonymous.second; ``` -This means the bound variables themselves are always references—they refer to the members of that hidden anonymous object. You cannot get the address of the "bound variable itself"; you can only get the address of the sub-object it references. +So the bound variables are always references—they refer to members of the hidden anonymous object. You can't take the address of "the bound variable itself"; you can only take the address of the sub-object it refers to. -⚠️ **Note:** `auto&` requires the right-hand side to be an lvalue. If the right-hand side is a temporary object (like the return value of a function), `auto&` will fail to compile because a non-const reference cannot bind to an rvalue. In this case, use `auto&&` or simply `auto` to copy by value. +⚠️ Note: `auto&` requires the right-hand side to be an lvalue. If the right-hand side is a temporary (like the return value of `std::make_pair(1, 2)`), `auto&` fails to compile—a non-const reference can't bind to an rvalue. Use `const auto&` or plain `auto` to copy instead. ```cpp -// auto& [x, y] = get_tuple(); // Error: cannot bind non-const lvalue reference to rvalue -auto&& [x, y] = get_tuple(); // OK: binds to rvalue reference +// Error: auto& can't bind to a temporary +auto& [x, y] = std::make_pair(1, 2); + +// OK: const reference extends the temporary's lifetime +const auto& [x, y] = std::make_pair(1, 2); + +// Or just copy +auto [x, y] = std::make_pair(1, 2); ``` ------ -## Step 4 — Custom Type Binding Support (Tuple-Like Protocol) +## Making custom types bindable: the tuple-like protocol -If your class has private members, it cannot be unpacked directly using the struct method. However, C++ offers another path: letting the compiler treat your class as a "tuple-like" type. You only need three things: +If your class has private members, you can't use the struct route. But C++ offers another path: tell the compiler to treat your class as a "tuple-like" type. You need three things: -1. Specialize `std::tuple_size` to tell the compiler how many elements there are. -2. Specialize `std::tuple_element` to tell the compiler the type of the `i`-th element. -3. Provide a `get` function in the same namespace as the class to return the `i`-th element. +1. Specialize `std::tuple_size` to tell the compiler how many elements there are. +2. Specialize `std::tuple_element` to tell it the type of the `I`-th element. +3. Provide a `get()` function in `YourType`'s namespace that returns the `I`-th element. ```cpp -struct MyElement { - int value; +#include +#include + +class SensorData { +public: + SensorData(uint8_t id, float value) : id_(id), value_(value) {} + + template + auto& get() { + if constexpr (I == 0) return id_; + else if constexpr (I == 1) return value_; + } + + template + const auto& get() const { + if constexpr (I == 0) return id_; + else if constexpr (I == 1) return value_; + } + +private: + uint8_t id_; + float value_; }; -struct MyContainer { - MyElement first; - MyElement second; -}; +// Specialize tuple_size: tell the compiler there are 2 elements +template<> +struct std::tuple_size : std::integral_constant {}; -// 1. Tell the compiler the size -template <> -struct std::tuple_size { - static constexpr size_t value = 2; -}; +// Specialize tuple_element: tell the compiler each element's type +template<> +struct std::tuple_element<0, SensorData> { using type = uint8_t; }; -// 2. Tell the compiler the type of each element -template -struct std::tuple_element { - using type = MyElement; -}; - -// 3. Provide get() function (ADL) -template -MyElement& get(MyContainer& c) { - if constexpr (I == 0) return c.first; - if constexpr (I == 1) return c.second; -} +template<> +struct std::tuple_element<1, SensorData> { using type = float; }; ``` -Now you can unpack it happily: +With ADL overloads for `get`, you can now happily unpack it: ```cpp -MyContainer c; -auto [a, b] = c; // a and b are MyElement +SensorData data{5, 23.5f}; +auto [id, value] = data; // id = 5, value = 23.5 +``` + +Run it to confirm (note `id` again needs `+` to print as a number): + +```text +id = 5, value = 23.5 ``` -> The key here is that the `get` function must be defined in the namespace where the class resides (ADL rule) so the compiler can find it. For specializations in the standard namespace `std`, you need to write specializations for `std::tuple_size` and `std::tuple_element` in the `std` namespace, but the `get` function can simply be placed in the class's namespace. +> The key here is that `get()` must be defined in the class's namespace (ADL rules) so the compiler can find it. For specializations that live in `std`, you write the `tuple_size` and `tuple_element` specializations inside `namespace std`, but the `get` function can stay in the class's namespace. -This mechanism is called the "tuple-like protocol." Standard library types like `std::pair`, `std::tuple`, and `std::array` rely on it to implement structured binding support. +This mechanism is called the "tuple-like protocol." The standard library's `std::pair`, `std::tuple`, and `std::array` all rely on it for structured binding support. ------ -## Enhancements in C++20 +## Changes in C++20 -C++20 made some enhancements to structured binding, mainly related to `constexpr` contexts. +C++20 made a few tweaks to structured binding, mostly around `constexpr` contexts. -Structured binding can be used inside `constexpr` functions, which means compile-time functions can also return multiple values and receive them via structured binding: +Structured binding can now be used inside `constexpr` functions, meaning compile-time computation can return multiple values and receive them via structured binding: ```cpp -constexpr auto split(int x) { - return std::tuple{x / 10, x % 10}; +constexpr auto get_point() { + return std::make_pair(3, 4); } -constexpr auto [div, mod] = split(42); // OK in C++20 +constexpr bool test_structured_binding() { + auto [x, y] = get_point(); + return x == 3 && y == 4; +} + +static_assert(test_structured_binding()); ``` -However, note that you cannot declare structured binding with `constexpr` directly at namespace scope (e.g., `constexpr auto [x, y] = ...;` is a compilation error). This is because structured binding is essentially a declaration of a set of reference variables, not a single variable declaration. +Note, though, that you can't declare a `constexpr` structured binding at namespace scope (e.g., `constexpr auto [x, y] = get_point();` is a compile error). That's because structured binding is fundamentally a declaration of a set of reference variables, not a single variable. -Regarding lambda captures, C++17 actually supports capturing structured binding variables directly. The following code works in C++17: +On lambda captures: C++17 already supports capturing structured-binding variables directly. This works in C++17: ```cpp -auto [x, y] = get_pair(); -auto f = [x, y] { return x + y; }; +std::map m = {{1, "one"}, {2, "two"}}; + +for (const auto& [k, v] : m) { + auto callback = [k, v] { // direct capture, valid in C++17 + std::cout << k << ": " << v << '\n'; + }; + callback(); +} ``` -C++20 added the init-capture syntax (e.g., `[x = x]`), which is more flexible in some cases. But be aware: default capture (`[=]` or `[&]`) does not automatically capture structured binding variables; you need to list them explicitly. +What C++20 adds is the init-capture syntax (`key = k`), which is more flexible in some cases. But note: `[=]` default capture does not capture structured-binding variables automatically—you have to list them explicitly. ------ -## Performance: Zero-Overhead Syntactic Sugar +## Performance: zero-overhead syntactic sugar -Structured binding itself has no runtime overhead. It is purely a compile-time syntactic transformation—the compiler creates an anonymous variable behind the scenes and then has the bound variables reference the anonymous variable's members. The generated assembly code is identical to hand-written code that "extracts members and assigns them." +Structured binding has no runtime overhead. It's purely a compile-time syntactic transformation—the compiler creates an anonymous variable behind the scenes and has the bound variables refer to its members. ```cpp -// These two generate the same assembly: +// These two generate identical assembly auto [x, y] = get_point(); -std::cout << x << y; -// vs -auto p = get_point(); -std::cout << p.x << p.y; +// equivalent to +auto __tmp = get_point(); +auto x = __tmp.first; +auto y = __tmp.second; ``` -Performance advice is simple: use `auto&` or `const auto&` for large structs to avoid copying, and use `auto` for small types (built-in types, small structs) to copy by value. `auto&&` is very useful in generic code, but when the specific type is known, explicitly writing `auto&` or `const auto&` is clearer. +"Identical assembly" is not a claim to make empty-handed. Compile both with GCC 16.1.1, `g++ -std=c++17 -O2 -S` each, then `diff`: + +```bash +g++ -std=c++17 -O2 -S sb_structured.cpp +g++ -std=c++17 -O2 -S sb_manual.cpp +diff sb_structured.s sb_manual.s +``` + +The `diff` output is a single line—the `.file` header differs (source filename); the actual instructions are identical: + +```text +_Z1fv: # f(), both versions identical + movl $7, %eax # returns 3 + 4 = 7 directly + ret +``` + +The compiler inlined `get_point()` and constant-folded it down to `movl $7, %eax`—structured binding left no trace. So the performance advice is simple: use `const auto&` for large structs to avoid copies, and `auto` to copy small types (built-ins, small structs). `auto&&` is useful in generic code, but when the concrete type is known, writing `auto` or `const auto&` explicitly is clearer. ------ -## Common Pitfalls +## Common pitfalls -### Lifetime Issues +### Lifetime issues -When `auto&&` or `auto` binds to a temporary object, the anonymous variable's lifetime is extended to the end of the binding variable's scope, so using `auto&&` or `auto` is safe. However, if you take a pointer or reference to the bound variable and pass it out, there is a risk of dangling: +When `auto&&` binds to a temporary, the anonymous variable's lifetime is extended to the end of the binding's scope, so `auto&&` and `const auto&` are safe. But if you take a pointer or reference to the bound variable and pass it out, you've got a dangling-reference risk: ```cpp -auto& [x, y] = get_temp_pair(); // get_temp_pair returns a temporary -// &x is a dangling reference after this line! +const auto& [x, y] = std::make_pair(1, 2); +// x, y are valid within this scope—safe. +// But if &x is stored outside, it dangles after the scope ends. ``` -### Cannot Be Used Directly as Return Values +### Can't be a return value directly -The variable names from structured binding cannot be used directly as function return values. If you want to return the unpacked values, you need to repack them: +Structured-binding names can't be used directly as a function return. If you want to return the unpacked values, you have to repack: ```cpp -auto [x, y] = get_pair(); -// return x, y; // Error -return std::make_pair(x, y); // OK +auto [x, y] = get_point(); +// can't: return x, y; must repack +return std::make_pair(x, y); + +// or just return the function's result +return get_point(); ``` -### Cannot Be Used for Class Member Declarations +### Can't be a class member declaration -You cannot use structured binding in class member declarations: +You can't use structured binding in a class member declaration: ```cpp -struct S { - auto [x, y] = get_pair(); // Error +class MyClass { + auto [x, y] = get_point(); // compile error }; ``` -If you need to store unpacked values, use a struct or `std::tuple`/`std::pair` members instead. +If you need to store unpacked values, use a struct or `pair`/`tuple` members instead. ------ -## Run Online +## Run online -Run the structured binding examples online to experience unpacking with `pair`, `tuple`, arrays, and structs: +Run the structured-binding examples online and see unpacking for `pair`, `tuple`, arrays, and structs: -## Summary +## Wrapping up -Structured binding is one of the most practical features in C++17. It covers the vast majority of daily development scenarios: `std::pair`, `std::tuple`, native arrays, structs with public members, and custom types implementing the tuple-like protocol. The binding semantics are entirely determined by the modifier before `auto`—`auto` is a copy, `auto&` is a reference, `const auto&` is a read-only reference, and `auto&&` is a forwarding reference. +That's the full coverage of types structured binding handles: `pair`, `tuple`, native arrays, structs with public members, plus custom types that implement the tuple-like protocol. The semantics are entirely decided by the modifier in front of `auto`—`auto` copies, `auto&` references, `const auto&` is read-only, `auto&&` forwards. -In practice, I most commonly use it for iterating over maps in range-based for loops (`const auto& [key, value]`) and handling multi-return functions. Combined with the if/switch initializers discussed in the next chapter, structured binding can take code conciseness and readability to the next level. +What I actually use day to day is range-based for over a map (`for (const auto& [k, v] : m)`) and catching multi-return functions. Pair it with the if/switch initializers in the next chapter and your code shrinks another size. ## References diff --git a/documents/en/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md b/documents/en/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md index 20a9ac180..808637d8f 100644 --- a/documents/en/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md +++ b/documents/en/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md @@ -91,10 +91,10 @@ sudo apt update Then install all the packages we need in one go: ```bash -sudo apt install gcc-arm-none-eabi gdb-arm-none-eabi openocd cmake build-essential +sudo apt install gcc-arm-none-eabi gdb-multiarch openocd cmake build-essential ``` -Let me explain what these packages do. `gcc-arm-none-eabi` is a big gift pack containing the cross-compiler, linker, `objcopy`, `size`, and a whole set of tools. `gdb-arm-none-eabi` is the ARM version of GDB for debugging embedded programs. `openocd` we mentioned earlier, for flashing and GDB Server. `cmake` and `build-essential` are build tools, with the latter containing basic compilation tools like `make`. +Let me explain what these packages do. `gcc-arm-none-eabi` is a big gift pack containing the cross-compiler, linker, `objcopy`, `size`, and a whole set of tools. `gdb-multiarch` is the multi-architecture GDB—it can debug ARM as well as RISC-V and other architectures. Note that the executable it installs is called `gdb-multiarch`, not `arm-none-eabi-gdb` (Ubuntu removed the legacy `gdb-arm-none-eabi` package from the repos a while ago). So wherever this tutorial says `arm-none-eabi-gdb`, Ubuntu users should substitute `gdb-multiarch` at the command line; Arch users keep `arm-none-eabi-gdb`. `openocd` we mentioned earlier, for flashing and GDB Server. `cmake` and `build-essential` are build tools, with the latter containing basic compilation tools like `make`. After installation, we can verify if the toolchain is actually installed: @@ -125,7 +125,7 @@ The installation command is a bit shorter than Ubuntu's: sudo pacman -S arm-none-eabi-gcc arm-none-eabi-binutils arm-none-eabi-gdb openocd cmake ``` -Here is a difference from Ubuntu: Arch splits the tools into multiple packages. `arm-none-eabi-gcc` is the compiler itself, `arm-none-eabi-binutils` contains `ld`, `objcopy`, `size`, and other tools, and `arm-none-eabi-gdb` is the debugger. Ubuntu packs all of these into `gcc-arm-none-eabi`, so fewer packages need to be installed. +Here is a difference from Ubuntu: Arch splits the tools into multiple packages. `arm-none-eabi-gcc` is the compiler itself, `arm-none-eabi-binutils` contains `ld`, `objcopy`, `size`, and other tools, and `arm-none-eabi-gdb` is the debugger. Ubuntu bundles the compiler and binutils into `gcc-arm-none-eabi`, while the debugger is the separate multi-architecture package `gdb-multiarch`—so the debugger command differs between the two distros (Arch: `arm-none-eabi-gdb`, Ubuntu: `gdb-multiarch`), as noted in the Ubuntu section above. Verify if the installation was successful: diff --git a/documents/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md b/documents/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md index 9735ba721..111a04afe 100644 --- a/documents/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md +++ b/documents/vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md @@ -20,15 +20,13 @@ title: 结构化绑定：一行解包多个值 --- # 结构化绑定：一行解包多个值 -笔者在写代码的时候，经常遇到一个很别扭的场景：函数返回了多个值，你得一个个拆开赋给变量。用 `pair` 的时候写 `result.first`、`result.second`，用 `tuple` 的时候写 `std::get<0>(t)` —— 要么语义不明，要么写法丑陋。C++11 引入了 `std::tie` 来缓解这个问题，但老实说，那语法也不算优雅：你得先声明好所有变量，再用 `tie` 往里塞。有没有那种,跟Python返回多个值的拆分写法一样爽的feature呢? 真有了孩子们! +笔者写代码时总撞上一个别扭的场景：函数返回多个值，你得一个个拆开赋给变量。用 `pair` 写 `result.first`、`result.second`，用 `tuple` 写 `std::get<0>(t)`——要么语义不明，要么写法丑陋。C++11 引入了 `std::tie` 缓解这事，但老实说那语法也不优雅：先声明好所有变量，再用 `tie` 往里塞。有没有那种跟 Python `a, b = func()` 一样爽的拆分写法？真有了，孩子们。 -C++17 终于给了我们一个真正的答案——结构化绑定（Structured Binding）。一行代码把 `pair`、`tuple`、数组、结构体全部拆开，直接拿到有名字的变量，语义清晰、零开销。 - -> 一句话总结：**结构化绑定让你把复合类型"解包"到多个命名变量中，编译器在幕后帮你完成一切。** +C++17 终于给了正经答案——结构化绑定（Structured Binding）。一行把 `pair`、`tuple`、数组、结构体全拆开，直接拿到有名字的变量，语义清晰，零开销。 ------ -## 第一步——绑定 pair 和 tuple +## 从 pair 和 tuple 讲起 ### pair：最常见的多返回值 @@ -64,7 +62,14 @@ for (const auto& [id, name] : sensor_names) { } ``` -> 为什么要写 `+id`？因为 `uint8_t` 的 `operator<<` 会把它当字符输出，`+` 会做整型提升（integral promotion），强制转换成 `int` 再打印。 +这里有个细节：循环体里写的是 `+id` 而不是 `id`。原因在于 `uint8_t` 的 `operator<<` 会把它当字符输出，而 `+` 会做整型提升（integral promotion），强制转换成 `int` 再打印。口说无凭，跑一下最直白（GCC 16.1.1，`-O2`）： + +```text +without + (raw uint8_t): A +with + (promoted) : 65 +``` + +同样是 `id = 65`，不加 `+` 打印出来是字符 `A`，加了才显示数字。 ### tuple：超过两个值的情况 @@ -93,7 +98,7 @@ std::tie(record_id, name, value) = query_database(42); ------ -## 第二步——绑定原生数组和结构体 +## 原生数组与结构体 ### 原生数组 @@ -107,7 +112,9 @@ auto [r, g, b] = rgb; 二维数组的每一行也可以在循环中解包： ```cpp -int matrix[2][3] = {{1, 2, 3}, {4, 5, 6}}; +int matrix[2][3] = { + {1, 2, 3}, {4, 5, 6} +}; for (auto& row : matrix) { auto [a, b, c] = row; std::cout << a << ' ' << b << ' ' << c << '\n'; @@ -132,13 +139,13 @@ SensorReading reading{5, 23.5f, 1234567890, true}; auto [id, val, ts, valid] = reading; ``` -这恐怕是结构化绑定最直观的用法了。你甚至不需要理解任何模板元编程，只要结构体成员是公有的就能用。 +不需要懂任何模板元编程，只要结构体成员是公有的就能用——这恐怕是结构化绑定最直观的用法了。 结构化绑定要求数据成员按声明顺序绑定，且完全支持位域（bit field）。如果结构体里有 `mutable` 成员，行为可能需要注意：绑定到的"匿名变量"可能被 `const` 修饰，但 `mutable` 成员不受此限制，仍可修改。 ------ -## 第三步——理解绑定的三种语义 +## 三种绑定语义 结构化绑定并不总是拷贝。实际上，`auto` 前面的修饰符决定了底层匿名变量的类型： @@ -147,7 +154,7 @@ auto [id, val, ts, valid] = reading; - **`const auto& [...]`**——绑定到 const 左值引用。只读访问，不拷贝。 - **`auto&& [...]`**——转发引用。既能绑定左值也能绑定右值。 -来看一个例子来区分它们： +用一个例子区分它们： ```cpp std::pair range{1, 10}; @@ -160,6 +167,13 @@ auto& [r3, r4] = range; r3 = 5; // range.first 变成 5 ``` +跑一下就能看到 `auto&` 改的是原对象、`auto` 改的是拷贝： + +```text +range.first after auto& mutation: 5 +r1 (copy, unaffected) : 1 +``` + 底层机制是这样的：编译器先声明一个匿名变量（类型由 `auto`/`auto&`/`const auto&`/`auto&&` 决定），用右侧表达式初始化它。然后每个绑定变量都是这个匿名变量的成员的引用（或者对于按值情况，是指向拷贝的成员的引用）。 ```cpp @@ -186,7 +200,7 @@ auto [x, y] = std::make_pair(1, 2); ------ -## 第四步——自定义类型的绑定支持（Tuple-Like Protocol） +## 让自定义类型支持绑定：Tuple-Like Protocol 如果你的类有私有成员，不能直接用结构体方式解绑。但 C++ 提供了另一条路：让编译器把你的类当作 "tuple-like" 类型来处理。只需要三样东西： @@ -231,22 +245,28 @@ template<> struct std::tuple_element<1, SensorData> { using type = float; }; ``` -现在就可以愉快地解包了： +配合 `get` 的 ADL 重载，现在就可以愉快地解包了： ```cpp SensorData data{5, 23.5f}; auto [id, value] = data; // id = 5, value = 23.5 ``` +实跑确认（注意 `id` 又得加 `+` 才打印成数字）： + +```text +id = 5, value = 23.5 +``` + > 这里的关键是 `get()` 函数必须定义在类所在的命名空间中（ADL 规则），这样编译器才能找到它。对于标准命名空间 `std` 中的特化，你需要在 `std` 命名空间中写 `tuple_size` 和 `tuple_element` 的特化，但 `get` 函数放在类所在的命名空间即可。 这套机制被称为 "tuple-like protocol"，标准库的 `std::pair`、`std::tuple`、`std::array` 都是靠它实现结构化绑定支持的。 ------ -## C++20 的增强 +## C++20 的变化 -C++20 对结构化绑定做了一些增强，主要与 constexpr 上下文相关。 +C++20 对结构化绑定做了几处调整，主要与 constexpr 上下文相关。 结构化绑定可以在 `constexpr` 函数内部使用，这意味着编译期计算函数也能返回多值并用结构化绑定接收： @@ -284,7 +304,7 @@ C++20 新增的是初始化捕获语法（`key = k`），这在某些情况下 ## 性能：零开销的语法糖 -结构化绑定本身没有任何运行时开销。它纯粹是编译期的语法变换——编译器会在幕后创建匿名变量，然后让绑定变量引用匿名变量的成员。生成的汇编代码和你手写的"取出成员再赋值"完全一致。 +结构化绑定本身没有任何运行时开销。它纯粹是编译期的语法变换——编译器会在幕后创建匿名变量，然后让绑定变量引用匿名变量的成员。 ```cpp // 这两种写法生成的汇编代码完全一样 @@ -296,7 +316,23 @@ auto x = __tmp.first; auto y = __tmp.second; ``` -性能方面的建议很简单：对于大结构体用 `const auto&` 避免拷贝，对于小型类型（内置类型、小 struct）直接用 `auto` 按值拷贝。`auto&&` 在泛型代码中很有用，但在具体类型已知的场景下，明确写 `auto` 或 `const auto&` 更清晰。 +"汇编完全一样"这种话不能空口说。拿 GCC 16.1.1 实测，两种写法各 `g++ -std=c++17 -O2 -S`，再 `diff`： + +```bash +g++ -std=c++17 -O2 -S sb_structured.cpp +g++ -std=c++17 -O2 -S sb_manual.cpp +diff sb_structured.s sb_manual.s +``` + +`diff` 输出只有一行——`.file` 那行的源文件名不同，实际指令完全一致： + +```text +_Z1fv: # f()，两个版本一模一样 + movl $7, %eax # 直接返回 3 + 4 = 7 + ret +``` + +编译器把 `get_point()` 内联后直接常量折叠成 `movl $7, %eax`，结构化绑定没留下任何痕迹。所以性能建议很简单：大结构体用 `const auto&` 避免拷贝，小类型（内置类型、小 struct）直接 `auto` 按值拷贝。`auto&&` 在泛型代码里有用，但具体类型已知时，明确写 `auto` 或 `const auto&` 更清晰。 ------ @@ -350,11 +386,11 @@ class MyClass { allow-run /> -## 小结 +## 收尾 -结构化绑定是 C++17 中最实用的特性之一。它支持的类型覆盖了日常开发的绝大部分场景：`pair`、`tuple`、原生数组、公有成员结构体，以及实现了 tuple-like protocol 的自定义类型。绑定的语义完全由 `auto` 前面的修饰符决定——`auto` 是拷贝，`auto&` 是引用，`const auto&` 是只读引用，`auto&&` 是转发引用。 +结构化绑定能覆盖的类型就这些：`pair`、`tuple`、原生数组、公有成员结构体，外加实现了 tuple-like protocol 的自定义类型。语义全由 `auto` 前面的修饰符决定——`auto` 拷贝、`auto&` 引用、`const auto&` 只读引用、`auto&&` 转发引用。 -在实战中，笔者最常用的是在范围 for 中遍历 map（`for (const auto& [k, v] : m)`）和处理多返回值函数。配合下一章要讲的 if/switch 初始化器，结构化绑定能让代码的简洁度和可读性都上一个台阶。 +笔者日常用得最多的还是范围 for 遍历 map（`for (const auto& [k, v] : m)`）和接多返回值函数。下一章的 if/switch 初始化器跟它配套，两者一起用，代码能再瘦一圈。 ## 参考资源 diff --git a/documents/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md b/documents/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md index 91b314f67..7407f5ba9 100644 --- a/documents/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md +++ b/documents/vol8-domains/embedded/00-env-setup/01-toolchain-setup.md @@ -86,13 +86,13 @@ sudo apt update ```bash sudo apt install -y \ gcc-arm-none-eabi \ - gdb-arm-none-eabi \ + gdb-multiarch \ openocd \ cmake \ build-essential ``` -让我解释一下这几个包都干嘛的。`gcc-arm-none-eabi` 是个大礼包，里面包含了交叉编译器、链接器、objcopy、size 等一整套工具。`gdb-arm-none-eabi` 是 ARM 版本的 GDB，用来调试嵌入式程序。`openocd` 我们前面说过了，是烧录和 GDB Server。`cmake` 和 `build-essential` 则是构建工具，后者包含了 make 等基础编译工具。 +让我解释一下这几个包都干嘛的。`gcc-arm-none-eabi` 是个大礼包，里面包含了交叉编译器、链接器、objcopy、size 等一整套工具。`gdb-multiarch` 是多架构版的 GDB，能调 ARM 也能调 RISC-V 之类的其他架构——注意它装出来的可执行文件就叫 `gdb-multiarch`，不是 `arm-none-eabi-gdb`（Ubuntu 早就把 `gdb-arm-none-eabi` 这个老包从源里移除了）。所以本教程里凡是写到 `arm-none-eabi-gdb` 的地方，Ubuntu 用户在命令行里替换成 `gdb-multiarch` 即可；Arch 用户保持 `arm-none-eabi-gdb`。`openocd` 我们前面说过了，是烧录和 GDB Server。`cmake` 和 `build-essential` 则是构建工具，后者包含了 make 等基础编译工具。 安装完成之后，我们可以验证一下工具链是不是真的装上了： @@ -122,7 +122,7 @@ This is free software; see the source for copying conditions. There is no warra sudo pacman -S arm-none-eabi-gcc arm-none-eabi-binutils arm-none-eabi-gdb openocd cmake make ``` -这里有个和 Ubuntu 不同的地方：Arch 把工具拆分成了多个包。`arm-none-eabi-gcc` 是编译器本身，`arm-none-eabi-binutils` 包含了 ld、objcopy、size 这些工具，`arm-none-eabi-gdb` 是调试器。Ubuntu 把这些都打包进了 `gcc-arm-none-eabi`，所以需要装的包更少。 +这里有个和 Ubuntu 不同的地方：Arch 把工具拆分成了多个包。`arm-none-eabi-gcc` 是编译器本身，`arm-none-eabi-binutils` 包含了 ld、objcopy、size 这些工具，`arm-none-eabi-gdb` 是调试器。Ubuntu 那边把编译器和 binutils 合在了 `gcc-arm-none-eabi` 里，调试器则是单独的多架构包 `gdb-multiarch`——所以两边调试器命令名不一样（Arch 是 `arm-none-eabi-gdb`，Ubuntu 是 `gdb-multiarch`），见前面 Ubuntu 段的说明。 验证一下安装是否成功： diff --git a/scripts/cppref_card_generator.py b/scripts/cppref_card_generator.py deleted file mode 100644 index 0c82ee4ab..000000000 --- a/scripts/cppref_card_generator.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python3 -""" -cppref_card_generator.py — 使用 Claude API 从爬取的 JSON 数据生成 C++ 特性参考卡。 - -用法: - python scripts/cppref_card_generator.py # 生成全部已抓取的特性 - python scripts/cppref_card_generator.py --features unique_ptr # 只生成指定特性 - python scripts/cppref_card_generator.py --dry-run # 只打印 prompt 不调用 API - -前置: - 1. 先运行 cppref_crawler.py 抓取内容 - 2. 设置 ANTHROPIC_API_KEY 环境变量 - -依赖: - pip install anthropic -""" - -import argparse -import json -import os -import sys -from pathlib import Path - -# 自动加载项目根目录的 .env 文件 -_PROJECT_ROOT = Path(__file__).resolve().parent.parent -_ENV_FILE = _PROJECT_ROOT / ".env" -if _ENV_FILE.exists(): - with open(_ENV_FILE, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line or line.startswith("#"): - continue - if "=" in line: - key, _, value = line.partition("=") - key = key.strip() - value = value.strip() - if key and key not in os.environ: - os.environ[key] = value - -try: - import anthropic -except ImportError: - print("错误: 需要安装依赖 — pip install anthropic", file=sys.stderr) - sys.exit(1) - -SCRIPT_DIR = Path(__file__).resolve().parent -PROJECT_ROOT = SCRIPT_DIR.parent -DEFAULT_CACHE_DIR = SCRIPT_DIR / "cppref_cache" -DEFAULT_MANIFEST = SCRIPT_DIR / "cppref_manifest.json" -TEMPLATE_PATH = None # 模板已内联，见 REFERENCE_CARD_TEMPLATE -WRITING_STYLE_PATH = PROJECT_ROOT / ".claude" / "style" / "writing-style.md" - -REFERENCE_CARD_TEMPLATE = """\ ---- -title: "特性名称" -description: "一句话摘要" -chapter: 99 -order: 0 -tags: - - host - - cpp-modern - - beginner -difficulty: beginner -cpp_standard: [11, 14, 17] ---- - -# 特性名称（C++XX） - -## 一句话 - -用一句人话说清楚这是什么、解决什么问题。 - -## 头文件 - -`#include <...>` - -## 核心 API 速查 - -| 操作 | 签名 | 说明 | -|------|------|------| -| ... | `...` | ... | - -## 最小示例 - -```cpp -// 完整可编译的最小示例，不超过 20 行 -// Standard: C++XX -``` - -## 嵌入式适用性：高/中/低 - -- 要点 1 -- 要点 2 - -## 编译器支持 - -| GCC | Clang | MSVC | -|-----|-------|------| -| X.Y | X.Y | 19.X | - -## 另见 - -- [教程：对应章节](相对路径) -- [cppreference: 特性名](https://en.cppreference.com/w/cpp/...) - ---- - -*部分内容参考自 [cppreference.com](https://en.cppreference.com/)，采用 [CC-BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) 许可* -""" - -SYSTEM_PROMPT = """\ -你是一个 C++ 参考文档生成助手。你的任务是根据 cppreference 的原始内容，生成符合以下模板的中文参考卡。 - -## 参考卡模板 - -{template} - -## 生成规则 - -1. **一句话**：用自然的中文描述这个特性是什么、解决什么问题，不要翻译 cppreference 的英文描述 -2. **头文件**：提取正确的 #include 指令 -3. **核心 API 速查表**：提取最常用的 5-10 个操作/函数，格式为表格。签名要准确，直接取自 cppreference -4. **最小示例**：写一个不超过 20 行、完整可编译的代码示例，展示最典型的用法 -5. **嵌入式适用性**：根据提供的评级（高/中/低），写 2-4 条要点分析在嵌入式开发中的适用性 -6. **编译器支持**：从 cppreference 的 raw_text 中提取 GCC/Clang/MSVC 的最低支持版本 -7. **另见**： - - 如果有 tutorial_link，写：`[教程：XXX](相对路径)` - - 写：`[cppreference: XXX](原始URL)` - -## 重要约束 - -- API 签名必须与 cppreference 一致，不要编造 -- 如果 raw_text 中找不到编译器支持信息，标注 "待补充" -- 不要写教程风格的叙事内容（不要踩坑叙述、不要"我们"人称） -- 保持参考卡的精炼结构化风格 -- frontmatter 中的 tags 从 manifest 中的信息推导 -""" - - -def build_frontmatter_tags(entry: dict) -> list[str]: - """根据 manifest 条目生成合法的 tags 列表。""" - tags = ["host", "cpp-modern"] - - # difficulty - diff = entry.get("difficulty", "beginner") - tags.append(diff) - - # 从 feature 名推导 tag - feature_lower = entry["feature"].lower() - tag_map = { - "unique_ptr": "unique_ptr", - "shared_ptr": "shared_ptr", - "optional": "optional", - "span": "span", - "variant": "variant", - "constexpr": "constexpr", - "lambda": "lambda", - "concepts": "concepts", - } - for key, tag in tag_map.items(): - if key in feature_lower: - tags.append(tag) - break - - # 智能指针类别 - if "ptr" in feature_lower and "智能指针" not in tags: - tags.append("智能指针") - - # 类型安全类别 - if feature_lower in ["std::optional", "std::variant"]: - if "类型安全" not in tags: - tags.append("类型安全") - - return tags - - -def build_user_prompt(entry: dict, crawled: dict) -> str: - """构建发送给 Claude 的用户 prompt。""" - content = crawled - - tutorial_note = "" - if entry.get("tutorial_link"): - tutorial_note = f"对应教程文章相对路径: {entry['tutorial_link']}" - - embedded_rating = entry.get("embedded_rating", "medium") - rating_desc = { - "high": "高 — 直接在嵌入式开发中使用，零开销或开销可控", - "medium": "中 — 可用但需注意开销，适合资源充足的场景", - "low": "低 — 通常在裸机嵌入式开发中避免", - } - - prompt = f"""\ -请根据以下 cppreference 原始内容，生成特性参考卡 markdown 文件。 - -## 特性信息 -- 特性名: {entry['feature']} -- 分类: {entry['category']} -- 序号: {entry['order']} -- C++ 标准: {entry['cpp_standard']} -- 难度: {entry['difficulty']} -- 嵌入式评级: {rating_desc.get(embedded_rating, embedded_rating)} -{tutorial_note} - -## cppreference 原始内容 - -### 标题 -{content.get('title', '')} - -### 头文件 -{content.get('header', '')} - -### 描述 -{content.get('description', '')} - -### 表格 -{chr(10).join(content.get('tables', []))} - -### 示例代码 -{chr(10).join('---' + chr(10) + ex for ex in content.get('examples', []))} - -### 全文（截取） -{content.get('raw_text', '')} - -### 另见链接 -{json.dumps(content.get('see_also', []), ensure_ascii=False, indent=2)} - ---- -请输出完整的 markdown 文件内容（包含 frontmatter），不要有任何其他解释。 -""" - return prompt - - -def load_crawled_data(cache_dir: Path, feature_name: str) -> dict | None: - """加载爬取的 JSON 数据。""" - from cppref_crawler import slugify - - slug = slugify(feature_name) - json_path = cache_dir / "raw" / f"{slug}.json" - if not json_path.exists(): - return None - with open(json_path, "r", encoding="utf-8") as f: - return json.load(f) - - -def generate_card( - client: anthropic.Anthropic, - entry: dict, - crawled_data: dict, - template: str, - model: str = "claude-sonnet-4-20250514", -) -> str: - """调用 Claude API 生成参考卡。""" - system_prompt = SYSTEM_PROMPT.format(template=template) - user_prompt = build_user_prompt(entry, crawled_data.get("crawled_content", {})) - - message = client.messages.create( - model=model, - max_tokens=4096, - system=system_prompt, - messages=[{"role": "user", "content": user_prompt}], - ) - - return message.content[0].text - - -def main(): - parser = argparse.ArgumentParser(description="使用 Claude API 生成 C++ 特性参考卡") - parser.add_argument( - "--manifest", - type=Path, - default=DEFAULT_MANIFEST, - help="manifest 文件路径", - ) - parser.add_argument( - "--cache-dir", - type=Path, - default=DEFAULT_CACHE_DIR, - help="爬取缓存目录", - ) - parser.add_argument( - "--features", - nargs="*", - help="只生成指定特性（按 feature 名匹配）", - ) - parser.add_argument( - "--model", - default=None, - help="Claude 模型（默认: 从 ANTHROPIC_DEFAULT_SONNET_MODEL 或 claude-sonnet-4-20250514）", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="只打印 prompt 不调用 API", - ) - args = parser.parse_args() - - # 加载模板 - template = REFERENCE_CARD_TEMPLATE - - # 加载 manifest - with open(args.manifest, "r", encoding="utf-8") as f: - manifest = json.load(f) - - # 过滤特性 - if args.features: - keywords = [k.lower() for k in args.features] - manifest = [ - entry - for entry in manifest - if any(kw in entry["feature"].lower() for kw in keywords) - ] - - if not manifest: - print("没有匹配的特性", file=sys.stderr) - sys.exit(1) - - # 检查缓存 - missing = [] - for entry in manifest: - from cppref_crawler import slugify - - slug = slugify(entry["feature"]) - cache_path = args.cache_dir / "raw" / f"{slug}.json" - if not cache_path.exists(): - missing.append(entry["feature"]) - - if missing: - print(f"警告: 以下特性尚未抓取，请先运行 cppref_crawler.py:") - for name in missing: - print(f" - {name}") - # 只处理已抓取的 - manifest = [ - entry for entry in manifest if entry["feature"] not in missing - ] - - if not manifest: - print("没有已抓取的特性可处理", file=sys.stderr) - sys.exit(1) - - # dry-run 模式 - if args.dry_run: - for entry in manifest: - crawled = load_crawled_data(args.cache_dir, entry["feature"]) - if crawled: - prompt = build_user_prompt(entry, crawled.get("crawled_content", {})) - print(f"\n{'='*60}") - print(f"特性: {entry['feature']}") - print(f"{'='*60}") - print(prompt[:2000]) - print("...(截断)") - return - - # 初始化 Claude 客户端（支持代理环境变量） - api_key = os.environ.get("ANTHROPIC_AUTH_TOKEN") or os.environ.get("ANTHROPIC_API_KEY") - if not api_key: - print("错误: 请设置 ANTHROPIC_AUTH_TOKEN 或 ANTHROPIC_API_KEY 环境变量", file=sys.stderr) - sys.exit(1) - - base_url = os.environ.get("ANTHROPIC_BASE_URL") - client_kwargs = {"api_key": api_key} - if base_url: - client_kwargs["base_url"] = base_url - client = anthropic.Anthropic(**client_kwargs) - - # 确定模型 - model = args.model or os.environ.get("ANTHROPIC_DEFAULT_SONNET_MODEL", "claude-sonnet-4-20250514") - print(f"准备生成 {len(manifest)} 张参考卡（模型: {model}）...") - - for entry in manifest: - feature_name = entry["feature"] - output_path = PROJECT_ROOT / entry["output_path"] - - print(f"\n生成: {feature_name}") - - crawled = load_crawled_data(args.cache_dir, feature_name) - if not crawled: - print(f" 跳过: 未找到缓存数据") - continue - - try: - card_md = generate_card(client, entry, crawled, template, model) - except anthropic.APIError as e: - print(f" API 错误: {e}", file=sys.stderr) - continue - - # 确保输出目录存在 - output_path.parent.mkdir(parents=True, exist_ok=True) - - # 清理可能的 markdown 代码块包裹 - card_md = card_md.strip() - if card_md.startswith("```markdown"): - card_md = card_md[len("```markdown"):] - if card_md.startswith("```"): - card_md = card_md[3:] - if card_md.endswith("```"): - card_md = card_md[:-3] - card_md = card_md.strip() - - with open(output_path, "w", encoding="utf-8") as f: - f.write(card_md + "\n") - - print(f" 已保存: {output_path.relative_to(PROJECT_ROOT)}") - - print(f"\n完成!") - - -if __name__ == "__main__": - main() diff --git a/scripts/cppref_manifest.json b/scripts/cppref_manifest.json deleted file mode 100644 index 1c0a1fe26..000000000 --- a/scripts/cppref_manifest.json +++ /dev/null @@ -1,442 +0,0 @@ -[ - { - "feature": "std::unique_ptr", - "cppref_url": "https://en.cppreference.com/w/cpp/memory/unique_ptr", - "output_path": "documents/cpp-reference/memory/01-unique-ptr.md", - "category": "memory", - "order": 1, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20], - "tutorial_link": "../vol2-modern-features/02-unique-ptr.md", - "embedded_rating": "high" - }, - { - "feature": "std::shared_ptr", - "cppref_url": "https://en.cppreference.com/w/cpp/memory/shared_ptr", - "output_path": "documents/cpp-reference/memory/02-shared-ptr.md", - "category": "memory", - "order": 2, - "difficulty": "intermediate", - "cpp_standard": [11, 14, 17, 20], - "tutorial_link": "../vol2-modern-features/03-shared-ptr.md", - "embedded_rating": "medium" - }, - { - "feature": "std::optional", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/optional", - "output_path": "documents/cpp-reference/memory/03-optional.md", - "category": "memory", - "order": 3, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": "../vol2-modern-features/04-optional.md", - "embedded_rating": "high" - }, - { - "feature": "std::span", - "cppref_url": "https://en.cppreference.com/w/cpp/container/span", - "output_path": "documents/cpp-reference/containers/01-span.md", - "category": "containers", - "order": 1, - "difficulty": "beginner", - "cpp_standard": [20, 23], - "tutorial_link": "../vol3-standard-library/containers/08-span.md", - "embedded_rating": "high" - }, - { - "feature": "std::string_view", - "cppref_url": "https://en.cppreference.com/w/cpp/string/basic_string_view", - "output_path": "documents/cpp-reference/containers/02-string-view.md", - "category": "containers", - "order": 2, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": "../vol2-modern-features/cpp17-string-view.md", - "embedded_rating": "high" - }, - { - "feature": "std::variant", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/variant", - "output_path": "documents/cpp-reference/containers/03-variant.md", - "category": "containers", - "order": 3, - "difficulty": "intermediate", - "cpp_standard": [17, 20, 23], - "tutorial_link": "../vol2-modern-features/03-variant.md", - "embedded_rating": "medium" - }, - { - "feature": "constexpr", - "cppref_url": "https://en.cppreference.com/w/cpp/language/constexpr", - "output_path": "documents/cpp-reference/core-language/01-constexpr.md", - "category": "core-language", - "order": 1, - "difficulty": "intermediate", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/03-constexpr.md", - "embedded_rating": "high" - }, - { - "feature": "lambda", - "cppref_url": "https://en.cppreference.com/w/cpp/language/lambda", - "output_path": "documents/cpp-reference/core-language/02-lambda.md", - "category": "core-language", - "order": 2, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/01-lambda-basics.md", - "embedded_rating": "high" - }, - { - "feature": "auto", - "cppref_url": "https://en.cppreference.com/w/cpp/language/auto", - "output_path": "documents/cpp-reference/core-language/03-auto-decltype.md", - "category": "core-language", - "order": 3, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/01-auto-and-decltype.md", - "embedded_rating": "high" - }, - { - "feature": "concepts", - "cppref_url": "https://en.cppreference.com/w/cpp/language/constraints", - "output_path": "documents/cpp-reference/templates/01-concepts.md", - "category": "templates", - "order": 1, - "difficulty": "intermediate", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "std::atomic", - "cppref_url": "https://en.cppreference.com/w/cpp/atomic/atomic", - "output_path": "documents/cpp-reference/concurrency/01-atomic.md", - "category": "concurrency", - "order": 1, - "difficulty": "intermediate", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol5-concurrency/ch03-atomic-memory-model/01-atomic-operations.md", - "embedded_rating": "high" - }, - { - "feature": "std::thread", - "cppref_url": "https://en.cppreference.com/w/cpp/thread/thread", - "output_path": "documents/cpp-reference/concurrency/02-thread.md", - "category": "concurrency", - "order": 2, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "std::mutex", - "cppref_url": "https://en.cppreference.com/w/cpp/thread/mutex", - "output_path": "documents/cpp-reference/concurrency/03-mutex.md", - "category": "concurrency", - "order": 3, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol5-concurrency/ch02-mutex-condition-sync/01-mutex-and-raii-guards.md", - "embedded_rating": "high" - }, - { - "feature": "nullptr", - "cppref_url": "https://en.cppreference.com/w/cpp/language/nullptr", - "output_path": "documents/cpp-reference/core-language/04-nullptr.md", - "category": "core-language", - "order": 4, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "enum class", - "cppref_url": "https://en.cppreference.com/w/cpp/language/enum", - "output_path": "documents/cpp-reference/core-language/05-enum-class.md", - "category": "core-language", - "order": 5, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/01-enum-class.md", - "embedded_rating": "high" - }, - { - "feature": "override", - "cppref_url": "https://en.cppreference.com/w/cpp/language/override", - "output_path": "documents/cpp-reference/core-language/06-override-final.md", - "category": "core-language", - "order": 6, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "range-for", - "cppref_url": "https://en.cppreference.com/w/cpp/language/range-for", - "output_path": "documents/cpp-reference/core-language/07-range-for.md", - "category": "core-language", - "order": 7, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/03-range-based-for-optimization.md", - "embedded_rating": "high" - }, - { - "feature": "std::move", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/move", - "output_path": "documents/cpp-reference/core-language/08-move-forward.md", - "category": "core-language", - "order": 8, - "difficulty": "intermediate", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/02-move-semantics.md", - "embedded_rating": "high" - }, - { - "feature": "generic lambda", - "cppref_url": "https://en.cppreference.com/w/cpp/language/lambda", - "output_path": "documents/cpp-reference/core-language/09-generic-lambda.md", - "category": "core-language", - "order": 9, - "difficulty": "intermediate", - "cpp_standard": [14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "std::exchange", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/exchange", - "output_path": "documents/cpp-reference/core-language/10-exchange.md", - "category": "core-language", - "order": 10, - "difficulty": "beginner", - "cpp_standard": [14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::make_unique", - "cppref_url": "https://en.cppreference.com/w/cpp/memory/unique_ptr/make_unique", - "output_path": "documents/cpp-reference/memory/04-make-unique.md", - "category": "memory", - "order": 4, - "difficulty": "beginner", - "cpp_standard": [14, 17, 20, 23], - "tutorial_link": "../vol2-modern-features/02-unique-ptr.md", - "embedded_rating": "high" - }, - { - "feature": "std::array", - "cppref_url": "https://en.cppreference.com/w/cpp/container/array", - "output_path": "documents/cpp-reference/containers/04-array.md", - "category": "containers", - "order": 4, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol3-standard-library/containers/02-array.md", - "embedded_rating": "high" - }, - { - "feature": "std::initializer_list", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/initializer_list", - "output_path": "documents/cpp-reference/containers/05-initializer-list.md", - "category": "containers", - "order": 5, - "difficulty": "beginner", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": "../vol3-standard-library/containers/11-initializer-lists.md", - "embedded_rating": "high" - }, - { - "feature": "variadic templates", - "cppref_url": "https://en.cppreference.com/w/cpp/language/parameter_pack", - "output_path": "documents/cpp-reference/templates/02-variadic-templates.md", - "category": "templates", - "order": 2, - "difficulty": "intermediate", - "cpp_standard": [11, 14, 17, 20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "fold expressions", - "cppref_url": "https://en.cppreference.com/w/cpp/language/fold", - "output_path": "documents/cpp-reference/templates/03-fold-expressions.md", - "category": "templates", - "order": 3, - "difficulty": "intermediate", - "cpp_standard": [17, 20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "structured bindings", - "cppref_url": "https://en.cppreference.com/w/cpp/language/structured_binding", - "output_path": "documents/cpp-reference/core-language/11-structured-binding.md", - "category": "core-language", - "order": 11, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": "../vol2-modern-features/ch05-structured-bindings/01-structured-bindings.md", - "embedded_rating": "high" - }, - { - "feature": "three-way comparison (<=>)", - "cppref_url": "https://en.cppreference.com/w/cpp/language/default_comparisons", - "output_path": "documents/cpp-reference/core-language/12-spaceship-operator.md", - "category": "core-language", - "order": 12, - "difficulty": "intermediate", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::jthread", - "cppref_url": "https://en.cppreference.com/w/cpp/thread/jthread", - "output_path": "documents/cpp-reference/concurrency/04-jthread.md", - "category": "concurrency", - "order": 4, - "difficulty": "beginner", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::expected", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/expected", - "output_path": "documents/cpp-reference/memory/05-expected.md", - "category": "memory", - "order": 5, - "difficulty": "intermediate", - "cpp_standard": [23], - "tutorial_link": "../vol2-modern-features/ch10-error-handling/03-expected-error.md", - "embedded_rating": "high" - }, - { - "feature": "if constexpr", - "cppref_url": "https://en.cppreference.com/w/cpp/language/if", - "output_path": "documents/cpp-reference/core-language/13-if-constexpr.md", - "category": "core-language", - "order": 13, - "difficulty": "intermediate", - "cpp_standard": [17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "inline variables", - "cppref_url": "https://en.cppreference.com/w/cpp/language/inline", - "output_path": "documents/cpp-reference/core-language/14-inline-variables.md", - "category": "core-language", - "order": 14, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": null, - "embedded_rating": "high" - }, - { - "feature": "nested namespaces", - "cppref_url": "https://en.cppreference.com/w/cpp/language/namespace", - "output_path": "documents/cpp-reference/core-language/15-nested-namespace.md", - "category": "core-language", - "order": 15, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": null, - "embedded_rating": "low" - }, - { - "feature": "std::filesystem", - "cppref_url": "https://en.cppreference.com/w/cpp/filesystem", - "output_path": "documents/cpp-reference/containers/06-filesystem.md", - "category": "containers", - "order": 6, - "difficulty": "beginner", - "cpp_standard": [17, 20, 23], - "tutorial_link": "../vol2-modern-features/ch09-filesystem/01-filesystem-path.md", - "embedded_rating": "low" - }, - { - "feature": "std::format", - "cppref_url": "https://en.cppreference.com/w/cpp/utility/format", - "output_path": "documents/cpp-reference/containers/07-format.md", - "category": "containers", - "order": 7, - "difficulty": "beginner", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::flat_map", - "cppref_url": "https://en.cppreference.com/w/cpp/container/flat_map", - "output_path": "documents/cpp-reference/containers/08-flat-map.md", - "category": "containers", - "order": 8, - "difficulty": "beginner", - "cpp_standard": [23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "coroutines", - "cppref_url": "https://en.cppreference.com/w/cpp/language/coroutines", - "output_path": "documents/cpp-reference/core-language/16-coroutines.md", - "category": "core-language", - "order": 16, - "difficulty": "intermediate", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "modules", - "cppref_url": "https://en.cppreference.com/w/cpp/language/modules", - "output_path": "documents/cpp-reference/core-language/17-modules.md", - "category": "core-language", - "order": 17, - "difficulty": "intermediate", - "cpp_standard": [20, 23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::generator", - "cppref_url": "https://en.cppreference.com/w/cpp/coroutine/generator", - "output_path": "documents/cpp-reference/containers/09-generator.md", - "category": "containers", - "order": 9, - "difficulty": "intermediate", - "cpp_standard": [23], - "tutorial_link": null, - "embedded_rating": "medium" - }, - { - "feature": "std::print", - "cppref_url": "https://en.cppreference.com/w/cpp/io/print", - "output_path": "documents/cpp-reference/containers/10-print.md", - "category": "containers", - "order": 10, - "difficulty": "beginner", - "cpp_standard": [23], - "tutorial_link": null, - "embedded_rating": "low" - }, - { - "feature": "deducing this", - "cppref_url": "https://en.cppreference.com/w/cpp/language/member_functions", - "output_path": "documents/cpp-reference/core-language/18-deducing-this.md", - "category": "core-language", - "order": 18, - "difficulty": "intermediate", - "cpp_standard": [23], - "tutorial_link": null, - "embedded_rating": "medium" - } -] diff --git a/scripts/fix_frontmatter.py b/scripts/fix_frontmatter.py deleted file mode 100644 index 0e853d6f4..000000000 --- a/scripts/fix_frontmatter.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix missing frontmatter in translated files. - -Reads frontmatter from the Chinese source file and injects it into the -corresponding English translation, translating only title and description. -Body content is left untouched. - -Usage: - python3 scripts/fix_frontmatter.py # Fix all missing frontmatter - python3 scripts/fix_frontmatter.py --dry-run # Show what would be fixed -""" - -import argparse -import json -import os -import re -import sys -from datetime import datetime, timezone -from pathlib import Path - -try: - import yaml -except ImportError: - print('ERROR: pyyaml is not installed.', file=sys.stderr) - print(' source .venv/bin/activate && pip install pyyaml', file=sys.stderr) - sys.exit(1) - -try: - import urllib.request - import urllib.error -except ImportError: - pass - -PROJECT_ROOT = Path(__file__).parent.parent -DOCS_DIR = PROJECT_ROOT / 'documents' -EN_DIR = PROJECT_ROOT / 'documents' / 'en' - -SUPPORTED_ENGINES = {'anthropic', 'openai'} - - -def load_api_key(engine: str) -> str: - """Load API key from environment or .env file.""" - engine_vars = { - 'anthropic': ['ANTHROPIC_AUTH_TOKEN'], - 'openai': ['OPENAI_API_KEY'], - } - for var in engine_vars.get(engine, []) + ['TRANSLATE_API_KEY']: - key = os.environ.get(var) - if key: - return key - - env_file = PROJECT_ROOT / '.env' - if env_file.exists(): - for line in env_file.read_text(encoding='utf-8').split('\n'): - line = line.strip() - if line.startswith('#') or '=' not in line: - continue - k, v = line.split('=', 1) - k = k.strip() - v = v.strip().strip('"').strip("'") - for var in engine_vars.get(engine, []) + ['TRANSLATE_API_KEY']: - if k == var: - return v - return None - - -def translate_text(text: str, api_key: str, engine: str) -> str: - """Translate a short text using the API.""" - base_url = os.environ.get('ANTHROPIC_BASE_URL', 'https://api.anthropic.com') \ - if engine == 'anthropic' else os.environ.get('OPENAI_BASE_URL', 'https://api.openai.com/v1') - model = os.environ.get('ANTHROPIC_DEFAULT_SONNET_MODEL', 'claude-sonnet-4-20250514') \ - if engine == 'anthropic' else 'gpt-4o-mini' - - prompt = f'Translate this to English. Output ONLY the translation, nothing else: {text}' - - if engine == 'anthropic': - payload = json.dumps({ - 'model': model, - 'max_tokens': 512, - 'messages': [{'role': 'user', 'content': prompt}], - 'temperature': 0.3, - }).encode('utf-8') - req = urllib.request.Request( - f'{base_url}/v1/messages', data=payload, - headers={ - 'Content-Type': 'application/json', - 'x-api-key': api_key, - 'anthropic-version': '2023-06-01', - }) - else: - payload = json.dumps({ - 'model': model, - 'messages': [{'role': 'user', 'content': prompt}], - 'temperature': 0.3, - }).encode('utf-8') - req = urllib.request.Request( - f'{base_url}/chat/completions', data=payload, - headers={ - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {api_key}', - }) - - resp = urllib.request.urlopen(req, timeout=60) - data = json.loads(resp.read().decode('utf-8')) - if engine == 'anthropic': - return data['content'][0]['text'].strip().strip('"') - else: - return data['choices'][0]['message']['content'].strip().strip('"') - - -def parse_frontmatter(content: str): - """Parse frontmatter. Returns (dict, frontmatter_str, body).""" - match = re.match(r'^---\s*\n(.*?)\n---\s*\n(.*)', content, re.DOTALL) - if not match: - return {}, '', content - try: - fm = yaml.safe_load(match.group(1)) - return fm or {}, match.group(1), match.group(2) - except Exception: - return {}, match.group(1), match.group(2) - - -def find_source_for(en_path: Path) -> Path: - """Find the Chinese source file for an English translation.""" - rel = en_path.relative_to(EN_DIR) - return DOCS_DIR / rel - - -def main(): - parser = argparse.ArgumentParser(description='Fix missing frontmatter in translated files') - parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') - parser.add_argument('--engine', choices=SUPPORTED_ENGINES, default='anthropic') - parser.add_argument('-y', '--yes', action='store_true', help='Skip confirmation') - args = parser.parse_args() - - # Find all en/ .md files missing frontmatter - missing = [] - for f in sorted(EN_DIR.rglob('*.md')): - content = f.read_text(encoding='utf-8') - fm, _, _ = parse_frontmatter(content) - if not fm: - missing.append(f) - - if not missing: - print('All translated files have frontmatter. Nothing to fix.') - return - - print(f'Found {len(missing)} files with missing frontmatter.') - if not args.dry_run and not args.yes: - confirm = input('Fix all? [y/N] ').strip().lower() - if confirm != 'y': - print('Aborted.') - return - - # Load API key for title/description translation - api_key = None - if not args.dry_run: - api_key = load_api_key(args.engine) - if not api_key: - print(f'ERROR: No API key for {args.engine}.', file=sys.stderr) - sys.exit(1) - - fixed = 0 - skipped = 0 - for en_path in missing: - src_path = find_source_for(en_path) - if not src_path.exists(): - print(f' SKIP {en_path.relative_to(PROJECT_ROOT)}: source not found') - skipped += 1 - continue - - src_content = src_path.read_text(encoding='utf-8') - src_fm, _, _ = parse_frontmatter(src_content) - if not src_fm: - print(f' SKIP {en_path.relative_to(PROJECT_ROOT)}: source also has no frontmatter') - skipped += 1 - continue - - en_content = en_path.read_text(encoding='utf-8') - _, _, en_body = parse_frontmatter(en_content) - - if args.dry_run: - print(f' [DRY RUN] {en_path.relative_to(PROJECT_ROOT)}') - fixed += 1 - continue - - # Translate title and description if they contain Chinese - merged_fm = dict(src_fm) - for field in ('title', 'description'): - val = merged_fm.get(field, '') - if val and re.search(r'[\u4e00-\u9fff]', val): - try: - merged_fm[field] = translate_text(val, api_key, args.engine) - except Exception as e: - print(f' WARN {en_path.name}: failed to translate {field}: {e}') - # Keep the Chinese value as fallback - - # Build output - fm_str = yaml.dump(merged_fm, allow_unicode=True, sort_keys=False, - default_flow_style=False).strip() - new_content = f'---\n{fm_str}\n---\n{en_body}' - en_path.write_text(new_content, encoding='utf-8') - print(f' OK {en_path.relative_to(PROJECT_ROOT)}') - fixed += 1 - - print(f'\nFixed: {fixed}, Skipped: {skipped}') - - -if __name__ == '__main__': - main() diff --git a/scripts/translate.py b/scripts/translate.py deleted file mode 100644 index 8765ffa45..000000000 --- a/scripts/translate.py +++ /dev/null @@ -1,1158 +0,0 @@ -#!/usr/bin/env python3 -""" -AI Translation Script — local incremental translation of Chinese Markdown docs. - -Translates Chinese .md files in documents/ to English, outputting to the -VitePress i18n folder structure (documents/en/). -Runs entirely locally, no CI involvement. - -Usage: - python3 scripts/translate.py --changed # Translate changed files vs main - python3 scripts/translate.py --file # Translate single file - python3 scripts/translate.py --dir # Translate all files under directory - python3 scripts/translate.py --all # Translate all (with confirmation) - python3 scripts/translate.py --changed --dry-run # Estimate cost only - python3 scripts/translate.py --changed --engine anthropic # Use Anthropic API - python3 scripts/translate.py --changed --force # Force re-translation - python3 scripts/translate.py --all --batch-size 20 # Translate max 20 files - python3 scripts/translate.py --all --workers 3 # Concurrent translation - -API key is read from: - 1. Environment variables (ANTHROPIC_AUTH_TOKEN / OPENAI_API_KEY / DEEPL_API_KEY) - 2. TRANSLATE_API_KEY generic fallback - 3. .env file in project root -""" - -import argparse -import hashlib -import json -import logging -import os -import re -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -# Ensure pyyaml is available (needed for frontmatter parsing) -try: - import yaml -except ImportError: - print('ERROR: pyyaml is not installed. Please activate the virtual environment first:', - file=sys.stderr) - print(' python3 -m venv .venv && source .venv/bin/activate && pip install pyyaml', - file=sys.stderr) - sys.exit(1) - -# --------------------------------------------------------------------------- -# Logging setup -# --------------------------------------------------------------------------- - -logger = logging.getLogger('translate') - - -def setup_logging(cache_dir: Path, verbose: bool = False) -> None: - """Configure dual-output logging: console (stderr) + file.""" - level = logging.DEBUG if verbose else logging.INFO - logger.setLevel(logging.DEBUG) - - # Console handler — stderr so it doesn't interfere with piping - console = logging.StreamHandler(sys.stderr) - console.setLevel(level) - console.setFormatter(logging.Formatter('%(levelname)s %(message)s')) - logger.addHandler(console) - - # File handler - cache_dir.mkdir(parents=True, exist_ok=True) - fh = logging.FileHandler(cache_dir / 'translate.log', encoding='utf-8') - fh.setLevel(logging.DEBUG) - fh.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s %(name)s %(message)s')) - logger.addHandler(fh) - - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- - -SUPPORTED_ENGINES = {'openai', 'anthropic', 'deepl'} -MAX_TOKENS_PER_RUN = 100_000 -CHARS_PER_TOKEN = 4 # rough estimate for mixed CJK/English -# Keep chunks small enough that the EN output stays under the 8192-token -# request cap (glm-4.6). English ≈ 1.8× CN chars, so ~0.45× CN chars in output -# tokens → must stay under ~16K CN chars/chunk. 60K caused silent tail truncation. -MAX_SINGLE_FILE_CHARS = 14_000 - -# VitePress i18n paths (relative to project root) -DOCS_DIR = 'documents' -I18N_OUTPUT_DIR = Path('documents') / 'en' - -# --------------------------------------------------------------------------- -# Terminology Glossary -# --------------------------------------------------------------------------- - -glossary_cache: Optional[Dict[str, str]] = None - - -def load_terminology_glossary(project_root: Path) -> Dict[str, str]: - """Parse terminology.md and return {Chinese: English} mapping.""" - global glossary_cache - if glossary_cache is not None: - return glossary_cache - - glossary_path = project_root / 'documents' / 'appendix' / 'terminology.md' - if not glossary_path.exists(): - logger.warning('Terminology glossary not found: %s', glossary_path) - glossary_cache = {} - return glossary_cache - - terms: Dict[str, str] = {} - content = glossary_path.read_text(encoding='utf-8') - # Parse markdown table rows: | English | 中文 | 备注 | - for line in content.split('\n'): - line = line.strip() - if not line.startswith('|') or line.startswith('|-') or line.startswith('| -'): - continue - cells = [c.strip() for c in line.split('|')] - # Filter empty strings from leading/trailing | - cells = [c for c in cells if c] - if len(cells) < 2: - continue - en_term = cells[0].strip() - zh_term = cells[1].strip() - # Skip header row and separator - if en_term.lower() == 'english' or set(en_term) <= {'-'}: - continue - if zh_term and en_term and re.search(r'[\u4e00-\u9fff]', zh_term): - terms[zh_term] = en_term - - glossary_cache = terms - logger.debug('Loaded %d terminology entries', len(terms)) - return terms - - -def format_terminology_for_prompt(terms: Dict[str, str]) -> str: - """Format terminology dict as a section for injection into system prompt.""" - if not terms: - return '' - lines = ['\n### Terminology Reference', '', - 'Use these established translations consistently:', - ''] - lines.append('| Chinese | English |') - lines.append('|---------|---------|') - for zh, en in sorted(terms.items(), key=lambda x: len(x[0]), reverse=True): - lines.append(f'| {zh} | {en} |') - lines.append('') - return '\n'.join(lines) - - -def build_system_prompt(project_root: Path) -> str: - """Build the full system prompt with terminology injection.""" - base_prompt = """You are a professional technical translator specializing in \ -modern C++ and embedded systems development. Your task is to translate Chinese \ -Markdown documentation into natural, idiomatic English. - -## Context -This is a tutorial project teaching modern C++ (C++11 through C++23) for \ -embedded systems (STM32, ESP32, RP2040). The content ranges from C++ language \ -features to hardware peripheral programming, RTOS concepts, and build systems. - -## Translation Rules - -### General -- Translate naturally, not literally. Prioritize readability for English-speaking developers. -- Maintain the exact Markdown structure and formatting. -- Use precise technical terminology consistently. - -### Content Preservation -- Do NOT translate code inside fenced code blocks (``` or ~~~). -- Do NOT translate inline code (`like this`). -- Translate code comments only if they contain Chinese text. -- Preserve the language specifier in fenced blocks (```cpp, ```python, etc.). -- Do NOT translate Mermaid diagram code blocks — keep them exactly as they are. -- Do NOT translate LaTeX math expressions ($...$ and $$...$$). -- When a fenced code block represents command output (not source code), use ```text as the language specifier instead of bare ```. - -### Tables -- Translate cell content, preserve table structure. -- Keep numeric values unchanged. - -### Images and Links -- In ![alt text](url), translate alt text but keep the URL unchanged. -- In [link text](url), translate link text but keep the URL unchanged. - -### Admonitions -- Translate admonition content. -- Keep admonition types (note, warning, tip, etc.) unchanged. - -### Frontmatter (YAML between ---) -- Translate ONLY the 'title' and 'description' fields. -- Keep all other fields (date, tags, difficulty, etc.) unchanged. -- The 'translation' metadata block must be kept as-is. - -### Style Guide -- Use active voice. -- Use "we" instead of "you" for a collaborative, hands-on tone. -- Keep sentences concise. -- Use serial comma (Oxford comma). -- Use American English spelling conventions. -- Spell out numbers below 10 (one, two, three). -""" - - terms = load_terminology_glossary(project_root) - terminology_section = format_terminology_for_prompt(terms) - return base_prompt + terminology_section - - -# --------------------------------------------------------------------------- -# Content Preprocessor — preserves code/mermaid/math/image URLs -# --------------------------------------------------------------------------- - -class ContentPreprocessor: - """Extract content that should not be translated and replace with placeholders.""" - - def extract_all(self, content: str) -> Tuple[str, Dict[str, str]]: - """Replace all preserved blocks with placeholders. - - Returns (modified_content, placeholder_dict). - """ - placeholders: Dict[str, str] = {} - counter = 0 - modified = content - - def _replace(m: re.Match) -> str: - nonlocal counter - key = f'__PRESERVED_{counter}__' - placeholders[key] = m.group(0) - counter += 1 - return key - - # 1. Fenced code blocks (``` and ~~~) - modified = re.sub(r'(`{3,}|~{3,})[\s\S]*?\1', _replace, modified) - - # 2. Display math $$...$$ - modified = re.sub(r'\$\$[\s\S]*?\$\$', _replace, modified) - - # 3. Inline math $...$ (not preceded/followed by $ or digit) - modified = re.sub(r'(? str: - """Restore preserved blocks from placeholders.""" - for key, value in placeholders.items(): - content = content.replace(key, value) - return content - - -# --------------------------------------------------------------------------- -# Translation Metadata -# --------------------------------------------------------------------------- - -@dataclass -class TranslationMetadata: - source: str = '' - source_hash: str = '' - translated_at: str = '' - engine: str = '' - token_count: int = 0 - - def to_dict(self) -> Dict: - return { - 'source': self.source, - 'source_hash': self.source_hash, - 'translated_at': self.translated_at, - 'engine': self.engine, - 'token_count': self.token_count, - } - - -# --------------------------------------------------------------------------- -# Frontmatter parsing & reconstruction -# --------------------------------------------------------------------------- - -def parse_frontmatter(content: str) -> Tuple[Dict, str, str]: - """Parse frontmatter, returns (dict, frontmatter_str, body).""" - match = re.match(r'^---\s*\n(.*?)\n---\s*\n(.*)', content, re.DOTALL) - if not match: - return {}, '', content - try: - fm = yaml.safe_load(match.group(1)) - return fm or {}, match.group(1), match.group(2) - except Exception: - return {}, match.group(1), match.group(2) - - -def compute_source_hash(content: str) -> str: - """Compute SHA-256 hash of file content.""" - return hashlib.sha256(content.encode('utf-8')).hexdigest() - - -def normalize_translated_title(title: str) -> str: - r"""Strip angle brackets the LLM re-adds around header names ( → numeric). - - VitePress renders sidebar titles via v-html, so raw < > get eaten as HTML - tags and the title's left side vanishes. The repo convention is bracket-free - titles (chrono / functional / optional are all bare), so drop < > entirely - and tidy any double spaces left behind. - """ - cleaned = re.sub(r'[<>]', '', title) - cleaned = re.sub(r'\s{2,}', ' ', cleaned).strip() - return cleaned - - -def reconstruct_frontmatter(fm: Dict, translated_fm: Dict, - metadata: Optional[TranslationMetadata] = None) -> str: - """Reconstruct frontmatter with translated fields and metadata.""" - merged = dict(fm) - # Overlay translated fields - if 'title' in translated_fm: - merged['title'] = normalize_translated_title(translated_fm['title']) - if 'description' in translated_fm: - merged['description'] = translated_fm['description'] - # Embed translation metadata - if metadata: - merged['translation'] = metadata.to_dict() - - # width=large keeps long scalars (especially the title) on a single line. - # The default (80) folds long titles across lines, and the sidebar title - # extractor only reads the first line — silently truncating the rest. - return yaml.dump(merged, allow_unicode=True, sort_keys=False, - default_flow_style=False, width=1_000_000).strip() - - -# --------------------------------------------------------------------------- -# Translation Manifest — content-hash based incremental detection -# --------------------------------------------------------------------------- - -class TranslationManifest: - """Manages .cache/translations/manifest.json for incremental translation.""" - - def __init__(self, cache_dir: Path): - self.cache_dir = cache_dir - self.manifest_path = cache_dir / 'manifest.json' - self.entries: Dict[str, Dict] = {} - - def load(self) -> None: - """Load manifest from disk.""" - if self.manifest_path.exists(): - try: - self.entries = json.loads( - self.manifest_path.read_text(encoding='utf-8')) - logger.debug('Loaded manifest with %d entries', len(self.entries)) - except (json.JSONDecodeError, OSError) as e: - logger.warning('Failed to load manifest: %s; starting fresh', e) - self.entries = {} - else: - self.entries = {} - - def save(self) -> None: - """Persist manifest to disk.""" - self.cache_dir.mkdir(parents=True, exist_ok=True) - self.manifest_path.write_text( - json.dumps(self.entries, indent=2, ensure_ascii=False), - encoding='utf-8') - - def is_up_to_date(self, rel_path: str, current_hash: str) -> bool: - """Check if translation exists and source hash matches.""" - entry = self.entries.get(rel_path) - if not entry: - return False - if entry.get('source_hash') != current_hash: - return False - return True - - def record(self, rel_path: str, source_hash: str, - engine: str, token_count: int) -> None: - """Record a successful translation.""" - self.entries[rel_path] = { - 'source_hash': source_hash, - 'translated_at': datetime.now(timezone.utc).isoformat(), - 'engine': engine, - 'token_count': token_count, - } - - def remove(self, rel_path: str) -> None: - """Remove an entry from the manifest.""" - self.entries.pop(rel_path, None) - - -# --------------------------------------------------------------------------- -# API clients -# --------------------------------------------------------------------------- - -class TranslationClient: - """Base class for translation API clients.""" - - def __init__(self, api_key: str, model: str = ''): - self.api_key = api_key - self.model = model - - def translate(self, text: str, system_prompt: str = '') -> str: - raise NotImplementedError - - def count_tokens(self, text: str) -> int: - return len(text) // CHARS_PER_TOKEN - - -class OpenAIClient(TranslationClient): - def __init__(self, api_key: str, model: str = 'gpt-4o-mini', base_url: str = ''): - super().__init__(api_key, model) - self.base_url = base_url or os.environ.get( - 'OPENAI_BASE_URL', 'https://api.openai.com/v1') - - def translate(self, text: str, system_prompt: str = '') -> str: - import urllib.request - import urllib.error - - url = f'{self.base_url}/chat/completions' - payload = json.dumps({ - 'model': self.model, - 'messages': [ - {'role': 'system', 'content': system_prompt}, - {'role': 'user', 'content': text}, - ], - 'temperature': 0.3, - }).encode('utf-8') - - req = urllib.request.Request( - url, - data=payload, - headers={ - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {self.api_key}', - }, - ) - - try: - resp = urllib.request.urlopen(req, timeout=120) - data = json.loads(resp.read().decode('utf-8')) - return data['choices'][0]['message']['content'] - except urllib.error.HTTPError as e: - body = e.read().decode('utf-8', errors='replace') - raise RuntimeError(f'API error {e.code}: {body}') - except KeyError: - raise RuntimeError(f'Unexpected API response: {data}') - - -class AnthropicClient(TranslationClient): - def __init__(self, api_key: str, - model: str = 'claude-sonnet-4-20250514', - base_url: str = ''): - super().__init__(api_key, model) - self.base_url = (base_url or - os.environ.get('ANTHROPIC_BASE_URL', - 'https://api.anthropic.com')) - - def translate(self, text: str, system_prompt: str = '') -> str: - import urllib.request - import urllib.error - - url = f'{self.base_url}/v1/messages' - payload = json.dumps({ - 'model': self.model, - 'max_tokens': 8192, - 'system': system_prompt, - 'messages': [ - {'role': 'user', 'content': text}, - ], - 'temperature': 0.3, - }).encode('utf-8') - - req = urllib.request.Request( - url, - data=payload, - headers={ - 'Content-Type': 'application/json', - 'x-api-key': self.api_key, - 'anthropic-version': '2023-06-01', - }, - ) - - try: - resp = urllib.request.urlopen(req, timeout=300) - data = json.loads(resp.read().decode('utf-8')) - return data['content'][0]['text'] - except urllib.error.HTTPError as e: - body = e.read().decode('utf-8', errors='replace') - raise RuntimeError(f'API error {e.code}: {body}') - except (KeyError, IndexError): - raise RuntimeError(f'Unexpected API response: {data}') - - -class DeepLClient(TranslationClient): - def __init__(self, api_key: str): - super().__init__(api_key) - - def translate(self, text: str, system_prompt: str = '') -> str: - import urllib.parse - import urllib.request - import urllib.error - - url = 'https://api-free.deepl.com/v2/translate' - payload = urllib.parse.urlencode({ - 'auth_key': self.api_key, - 'text': text, - 'source_lang': 'ZH', - 'target_lang': 'EN', - }).encode('utf-8') - - req = urllib.request.Request(url, data=payload) - try: - resp = urllib.request.urlopen(req, timeout=120) - data = json.loads(resp.read().decode('utf-8')) - return data['translations'][0]['text'] - except urllib.error.HTTPError as e: - body = e.read().decode('utf-8', errors='replace') - raise RuntimeError(f'API error {e.code}: {body}') - - -def create_client(engine: str, api_key: str) -> TranslationClient: - if engine == 'openai': - return OpenAIClient(api_key) - elif engine == 'anthropic': - model = os.environ.get('ANTHROPIC_DEFAULT_SONNET_MODEL', - 'claude-sonnet-4-20250514') - return AnthropicClient(api_key, model=model) - elif engine == 'deepl': - return DeepLClient(api_key) - else: - raise ValueError(f'Unsupported engine: {engine}') - - -# --------------------------------------------------------------------------- -# Large file splitting -# --------------------------------------------------------------------------- - -def split_body_by_headings(body: str, max_chars: int = MAX_SINGLE_FILE_CHARS - ) -> List[str]: - """Split body text at ## heading boundaries to keep each chunk under max_chars. - - Returns a list of body chunks. Each chunk starts with its heading. - If the body fits in one chunk, returns [body]. - """ - if len(body) <= max_chars: - return [body] - - # Split at ## headings (but not # which is the document title) - parts = re.split(r'(?=^## )', body, flags=re.MULTILINE) - - # Reassemble into chunks that fit within max_chars - chunks: List[str] = [] - current = '' - for part in parts: - if not current: - current = part - elif len(current) + len(part) <= max_chars: - current += part - else: - if current.strip(): - chunks.append(current) - current = part - if current.strip(): - chunks.append(current) - - if len(chunks) > 1: - logger.debug('Split into %d chunks (max %d chars each)', - len(chunks), max_chars) - return chunks if chunks else [body] - - -# --------------------------------------------------------------------------- -# Translation Pipeline -# --------------------------------------------------------------------------- - -class TranslationPipeline: - def __init__(self, client: TranslationClient, - manifest: TranslationManifest, - preprocessor: ContentPreprocessor, - system_prompt: str, - engine_name: str, - dry_run: bool = False, - force: bool = False, - max_tokens: int = MAX_TOKENS_PER_RUN, - project_root: Path = None): - self.client = client - self.manifest = manifest - self.preprocessor = preprocessor - self.system_prompt = system_prompt - self.engine_name = engine_name - self.dry_run = dry_run - self.force = force - self.max_tokens = max_tokens - self.project_root = project_root - self.tokens_used = 0 - self.files_translated = 0 - self.files_skipped = 0 - self.files_failed = 0 - - def should_translate(self, rel_path: str, source_hash: str) -> bool: - """Check if file needs translation based on content hash.""" - if self.force: - return True - if self.manifest.is_up_to_date(rel_path, source_hash): - logger.info('SKIP %s: source hash unchanged (%s)', rel_path, - source_hash[:12]) - self.files_skipped += 1 - return False - return True - - # Fenced code blocks (``` / ~~~) and display math ($$…$$) are preserved - # verbatim; everything between them is prose sent to the model. - _PRESERVE_RE = re.compile(r'(`{3,}|~{3,})[\s\S]*?\1|\$\$[\s\S]*?\$\$') - - def _translate_body_prose_only(self, body: str) -> str: - """Translate a document body, keeping fenced code & display math verbatim. - - Splits the body into alternating prose and code/math segments. Code and - math are stitched back unchanged; each prose run is translated as a - coherent unit (so glm-4.6 doesn't hallucinate fences or meta-commentary, - which it does on tiny fragments). A prose run larger than the chunk - limit is further split at ## heading boundaries first, keeping each - request under the output-token cap. - """ - segments: List[Tuple[str, str]] = [] - last = 0 - for m in self._PRESERVE_RE.finditer(body): - if m.start() > last: - segments.append(('prose', body[last:m.start()])) - segments.append(('code', m.group(0))) - last = m.end() - if last < len(body): - segments.append(('prose', body[last:])) - - out: List[str] = [] - for seg_type, seg in segments: - if seg_type == 'code': - out.append(seg) - continue - if not seg.strip(): - continue # whitespace-only glue; the '\n\n' join re-adds spacing - chunks = (split_body_by_headings(seg) - if len(seg) > MAX_SINGLE_FILE_CHARS else [seg]) - for chunk in chunks: - try: - # strip(): the model drops the leading/trailing newlines that - # separated this prose from adjacent code; re-join with blank - # lines below so fences open/close on their own lines. - out.append(self.client.translate(chunk, self.system_prompt).strip()) - except Exception as e: - logger.error('FAILED translating prose segment: %s', e) - raise - return '\n\n'.join(out) - - def translate_file(self, input_path: Path, - project_root: Path) -> Optional[str]: - """Translate a single file. Returns translated content or None.""" - rel_path = str(input_path.relative_to(project_root)) - - try: - content = input_path.read_text(encoding='utf-8') - except Exception as e: - logger.error('Error reading %s: %s', rel_path, e) - self.files_failed += 1 - return None - - # Compute source hash - source_hash = compute_source_hash(content) - - # Check incremental — skip if unchanged - if not self.should_translate(rel_path, source_hash): - return None - - # Skip non-Chinese files - chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', content)) - if chinese_chars < 5: - logger.info('SKIP %s: insufficient Chinese content (%d CJK chars)', - rel_path, chinese_chars) - self.files_skipped += 1 - return None - - # Estimate tokens - estimated_tokens = self.client.count_tokens(content) - if self.max_tokens > 0 and self.tokens_used + estimated_tokens > self.max_tokens: - logger.info( - 'SKIP %s: would exceed token limit (%d + %d > %d)', - rel_path, self.tokens_used, estimated_tokens, self.max_tokens) - self.files_skipped += 1 - return None - - if self.dry_run: - cost = self._estimate_cost(estimated_tokens) - logger.info('[DRY RUN] %s: ~%d tokens, ~$%.4f', - rel_path, estimated_tokens, cost) - self.tokens_used += estimated_tokens - return None - - # Parse frontmatter - fm, fm_str, body = parse_frontmatter(content) - - # Translate body, preserving fenced code & display math verbatim. - # glm-4.6 strips __PRESERVED_N__ placeholders and regenerates code - # (often dropping or altering blocks in code-heavy files), so instead - # we split the body at code/math boundaries and translate only the - # prose runs — each kept coherent (no mid-sentence fragmentation) — - # and stitch code back unchanged. - translated_body = self._translate_body_prose_only(body) - - # Translate frontmatter title and description - translated_fm: Dict[str, str] = {} - if fm: - title = fm.get('title', '') - if title and re.search(r'[\u4e00-\u9fff]', title): - try: - translated_fm['title'] = self.client.translate( - f'Translate this title to English: {title}', - self.system_prompt - ).strip().strip('"') - except Exception: - translated_fm['title'] = title - desc = fm.get('description', '') - if desc and re.search(r'[\u4e00-\u9fff]', desc): - try: - translated_fm['description'] = self.client.translate( - f'Translate this description to English: {desc}', - self.system_prompt - ).strip().strip('"') - except Exception: - translated_fm['description'] = desc - - # Build translation metadata - metadata = TranslationMetadata( - source=rel_path, - source_hash=source_hash, - translated_at=datetime.now(timezone.utc).isoformat(), - engine=self.engine_name, - token_count=estimated_tokens, - ) - - # Reconstruct document - if fm: - new_fm = reconstruct_frontmatter(fm, translated_fm, metadata) - result = f'---\n{new_fm}\n---\n{translated_body}' - else: - result = translated_body - - self.tokens_used += estimated_tokens - self.files_translated += 1 - - # Update manifest - self.manifest.record(rel_path, source_hash, - self.engine_name, estimated_tokens) - - logger.info('OK %s', rel_path) - return result - - def _estimate_cost(self, tokens: int) -> float: - return tokens / 1000 * 0.01 - - -# --------------------------------------------------------------------------- -# Post-translation validation -# --------------------------------------------------------------------------- - -def validate_translation(output_path: Path) -> List[str]: - """Validate a translated file. Returns list of warnings.""" - warnings: List[str] = [] - try: - content = output_path.read_text(encoding='utf-8') - except Exception as e: - return [f'Cannot read output: {e}'] - - # Check for leftover placeholders - leftover = re.findall(r'__PRESERVED_\d+__', content) - if leftover: - warnings.append(f'{len(leftover)} leftover __PRESERVED_ placeholders') - - # Check frontmatter is parseable - fm, fm_str, body = parse_frontmatter(content) - if fm_str and not fm: - warnings.append('Frontmatter exists but failed to parse as YAML') - - # Check translation metadata exists - if fm and 'translation' not in fm: - warnings.append('Missing translation metadata in frontmatter') - - return warnings - - -# --------------------------------------------------------------------------- -# Change detection -# --------------------------------------------------------------------------- - -def detect_changed_files(project_root: Path) -> List[Path]: - """Detect changed .md files in documents/ relative to main branch.""" - try: - result = subprocess.run( - ['git', 'diff', '--name-only', '--diff-filter=ACM', 'main...HEAD', - '--', f'{DOCS_DIR}/**/*.md'], - cwd=str(project_root), - capture_output=True, - text=True, - ) - files = [] - for line in result.stdout.strip().split('\n'): - if not line: - continue - path = project_root / line - # Skip files inside the en/ output directory - try: - rel = path.relative_to(project_root / DOCS_DIR) - if rel.parts and rel.parts[0] == 'en': - continue - except ValueError: - pass - if path.exists() and path.suffix == '.md': - files.append(path) - return files - except Exception as e: - logger.warning('Could not detect changed files: %s', e) - return [] - - -def find_all_chinese_md(docs_root: Path) -> List[Path]: - """Find all Chinese .md files in documents/ (excluding en/ output directory).""" - files = [] - for f in docs_root.rglob('*.md'): - # Skip files inside the en/ i18n output directory - try: - rel = f.relative_to(docs_root) - if rel.parts and rel.parts[0] == 'en': - continue - except ValueError: - pass - # Skip non-content directories - if any(part in ('images', 'generated', 'hooks', 'stylesheets', - 'javascripts') for part in f.parts): - continue - files.append(f) - return sorted(files) - - -def get_output_path(input_path: Path, project_root: Path) -> Path: - """Compute VitePress i18n output path for a given source file. - - Input: documents/vol1-fundamentals/00-preface.md - Output: documents/en/vol1-fundamentals/00-preface.md - """ - rel = input_path.relative_to(project_root / DOCS_DIR) - return project_root / I18N_OUTPUT_DIR / rel - - -# --------------------------------------------------------------------------- -# API Key loading -# --------------------------------------------------------------------------- - -def load_api_key(engine: str) -> Optional[str]: - """Load API key from environment or .env file.""" - engine_vars = { - 'anthropic': ['ANTHROPIC_AUTH_TOKEN'], - 'openai': ['OPENAI_API_KEY'], - 'deepl': ['DEEPL_API_KEY'], - } - fallback_vars = ['TRANSLATE_API_KEY'] - - for var in engine_vars.get(engine, []) + fallback_vars: - key = os.environ.get(var) - if key: - return key - - env_file = Path(__file__).parent.parent / '.env' - if env_file.exists(): - for line in env_file.read_text(encoding='utf-8').split('\n'): - line = line.strip() - if line.startswith('#') or '=' not in line: - continue - k, v = line.split('=', 1) - k = k.strip() - v = v.strip().strip('"').strip("'") - for var in engine_vars.get(engine, []) + fallback_vars: - if k == var: - return v - - return None - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - parser = argparse.ArgumentParser( - description='Translate Chinese Markdown docs to English (VitePress i18n)') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--changed', action='store_true', - help='Translate files changed relative to main') - group.add_argument('--file', type=str, - help='Translate a single file (relative to project root)') - group.add_argument('--dir', type=str, - help='Translate all .md files under a directory (relative to project root)') - group.add_argument('--all', action='store_true', - help='Translate all Chinese .md files in documents/') - parser.add_argument('--engine', choices=SUPPORTED_ENGINES, - default='anthropic', help='Translation engine (default: anthropic)') - parser.add_argument('--dry-run', action='store_true', - help='Estimate tokens and cost without translating') - parser.add_argument('--force', action='store_true', - help='Force re-translation regardless of content hash') - parser.add_argument('--max-tokens', type=int, default=0, - help='Max tokens per run (default: unlimited)') - parser.add_argument('--batch-size', type=int, default=0, - help='Max files to translate per run (default: unlimited)') - parser.add_argument('--delay', type=float, default=2, - help='Delay in seconds between files (default: 2)') - parser.add_argument('--retries', type=int, default=3, - help='Retry attempts on API errors (default: 3)') - parser.add_argument('--workers', type=int, default=1, - help='Concurrent translation workers (default: 1)') - parser.add_argument('-v', '--verbose', action='store_true', - help='Enable debug-level logging') - parser.add_argument('-y', '--yes', action='store_true', - help='Skip confirmation prompt') - args = parser.parse_args() - - project_root = Path(__file__).parent.parent - docs_root = project_root / DOCS_DIR - cache_dir = project_root / '.cache' / 'translations' - - # Setup logging - setup_logging(cache_dir, args.verbose) - logger.info('Translation run started (engine=%s, dry_run=%s, force=%s)', - args.engine, args.dry_run, args.force) - - # Collect files to translate - if args.file: - target = Path(args.file) - if not target.is_absolute(): - target = project_root / target - if not target.exists(): - logger.error('File not found: %s', target) - sys.exit(1) - files = [target] - elif args.dir: - target = Path(args.dir) - if not target.is_absolute(): - target = project_root / target - if not target.is_dir(): - logger.error('Directory not found: %s', target) - sys.exit(1) - files = find_all_chinese_md(target) - if not files: - logger.info('No Chinese .md files found in %s.', target) - sys.exit(0) - logger.info('Found %d Chinese .md files in %s.', len(files), target) - if not args.dry_run and not args.yes: - confirm = input(f'Translate {len(files)} files? [y/N] ').strip().lower() - if confirm != 'y': - logger.info('Aborted.') - sys.exit(0) - elif args.changed: - files = detect_changed_files(project_root) - if not files: - logger.info('No changed Chinese .md files detected.') - sys.exit(0) - logger.info('Detected %d changed file(s):', len(files)) - for f in files: - logger.info(' %s', f.relative_to(project_root)) - elif args.all: - files = find_all_chinese_md(docs_root) - if not files: - logger.info('No Chinese .md files found in %s.', docs_root) - sys.exit(0) - logger.info('Found %d Chinese .md files.', len(files)) - if not args.dry_run and not args.yes: - confirm = input('Translate all? [y/N] ').strip().lower() - if confirm != 'y': - logger.info('Aborted.') - sys.exit(0) - - # Load API key - if not args.dry_run: - api_key = load_api_key(args.engine) - if not api_key: - logger.error('No API key found for %s.', args.engine) - print(f'\nSet one of:', file=sys.stderr) - if args.engine == 'anthropic': - print(' export ANTHROPIC_AUTH_TOKEN=\'your-key\'', file=sys.stderr) - elif args.engine == 'openai': - print(' export OPENAI_API_KEY=\'your-key\'', file=sys.stderr) - print(' export TRANSLATE_API_KEY=\'your-key\'', file=sys.stderr) - print(' Or add to .env (see .env.example)', file=sys.stderr) - sys.exit(1) - client = create_client(args.engine, api_key) - else: - client = create_client(args.engine, 'dummy') - - # Initialize components - manifest = TranslationManifest(cache_dir) - manifest.load() - - preprocessor = ContentPreprocessor() - system_prompt = build_system_prompt(project_root) - - pipeline = TranslationPipeline( - client=client, - manifest=manifest, - preprocessor=preprocessor, - system_prompt=system_prompt, - engine_name=args.engine, - dry_run=args.dry_run, - force=args.force, - max_tokens=args.max_tokens, - project_root=project_root, - ) - - # Translate - import time - import threading - from concurrent.futures import ThreadPoolExecutor, as_completed - - total_files = len(files) - batch_count = 0 - batch_lock = threading.Lock() - done_counter = [0] - done_lock = threading.Lock() - - def translate_one(filepath: Path) -> str: - """Translate a single file with retry logic. Returns status string.""" - nonlocal batch_count - rel = filepath.relative_to(project_root) - output_path = get_output_path(filepath, project_root) - - with done_lock: - done_counter[0] += 1 - progress = f'[{done_counter[0]}/{total_files}]' - - # Retry loop - result = None - for attempt in range(1, args.retries + 1): - try: - result = pipeline.translate_file(filepath, project_root) - break - except Exception as e: - if attempt < args.retries: - wait = args.delay + attempt * 3 - logger.warning('%s %s attempt %d/%d failed (%s), retrying in %.0fs...', - progress, rel, attempt, args.retries, str(e)[:80], wait) - time.sleep(wait) - else: - logger.error('%s FAILED %s: %s', progress, rel, e) - return 'failed' - - if result is None: - return 'skipped' - - try: - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(result.rstrip('\n') + '\n', - encoding='utf-8') - logger.info('%s Written: %s', progress, output_path.relative_to(project_root)) - - warns = validate_translation(output_path) - for w in warns: - logger.warning('Validation %s: %s', rel, w) - except Exception as e: - logger.error('Error writing %s: %s', output_path, e) - return 'failed' - - with batch_lock: - batch_count += 1 - - # Delay between requests (per worker) - if args.delay > 0: - time.sleep(args.delay) - - return 'ok' - - if args.dry_run: - for filepath in files: - pipeline.translate_file(filepath, project_root) - elif args.workers > 1: - logger.info('Starting %d workers for %d files...', args.workers, total_files) - with ThreadPoolExecutor(max_workers=args.workers) as executor: - futures = {executor.submit(translate_one, f): f for f in files} - for future in as_completed(futures): - status = future.result() - if status == 'failed': - pipeline.files_failed += 1 - if args.batch_size > 0: - with batch_lock: - if batch_count >= args.batch_size: - logger.info('Batch limit reached (%d files). Stopping.', - args.batch_size) - executor.shutdown(wait=False, cancel_futures=True) - break - else: - # Single-threaded mode with progress - for idx, filepath in enumerate(files, 1): - progress = f'[{idx}/{total_files}]' - rel = filepath.relative_to(project_root) - output_path = get_output_path(filepath, project_root) - - result = None - for attempt in range(1, args.retries + 1): - try: - result = pipeline.translate_file(filepath, project_root) - break - except Exception as e: - if attempt < args.retries: - wait = args.delay + attempt * 3 - logger.warning('%s %s attempt %d/%d failed (%s), retrying in %.0fs...', - progress, rel, attempt, args.retries, str(e)[:80], wait) - time.sleep(wait) - else: - logger.error('%s FAILED %s: %s', progress, rel, e) - pipeline.files_failed += 1 - result = None - break - - if result is not None: - try: - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(result.rstrip('\n') + '\n', - encoding='utf-8') - logger.info('%s Written: %s', progress, output_path.relative_to(project_root)) - - warns = validate_translation(output_path) - for w in warns: - logger.warning('Validation %s: %s', rel, w) - - batch_count += 1 - except Exception as e: - logger.error('Error writing %s: %s', output_path, e) - pipeline.files_failed += 1 - - if args.delay > 0 and idx < total_files: - time.sleep(args.delay) - - if args.batch_size > 0 and batch_count >= args.batch_size: - logger.info('Batch limit reached (%d files). Stopping.', - args.batch_size) - break - - manifest.save() - - # Summary - logger.info('=' * 50) - if args.dry_run: - logger.info('Estimated tokens: %s', f'{pipeline.tokens_used:,}') - cost = pipeline._estimate_cost(pipeline.tokens_used) - logger.info('Estimated cost: ~$%.2f', cost) - logger.info('Files: %d', len(files)) - else: - logger.info('Files translated: %d', pipeline.files_translated) - logger.info('Files skipped: %d', pipeline.files_skipped) - logger.info('Files failed: %d', pipeline.files_failed) - logger.info('Tokens used: %s', f'{pipeline.tokens_used:,}') - logger.info('=' * 50) - - sys.exit(1 if pipeline.files_failed > 0 and not args.dry_run else 0) - - -if __name__ == '__main__': - main()