diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..8c25686 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,14 @@ +{ + "pins" : [ + { + "identity" : "swift-tiktoken", + "kind" : "remoteSourceControl", + "location" : "https://github.com/DePasqualeOrg/swift-tiktoken.git", + "state" : { + "revision" : "e088d534d80dd1875ca3edfd69a16a6c25ab84c4", + "version" : "0.0.1" + } + } + ], + "version" : 2 +} diff --git a/Package.swift b/Package.swift index ae825a3..d670471 100644 --- a/Package.swift +++ b/Package.swift @@ -1,33 +1,34 @@ -// swift-tools-version: 5.9 +// swift-tools-version:5.9 import PackageDescription let package = Package( name: "TiktokenSwift", platforms: [ - .macOS(.v10_15), - .iOS(.v13), - .tvOS(.v13), - .watchOS(.v6) + .macOS(.v14), + .iOS(.v16), + .tvOS(.v16), + .watchOS(.v9) ], products: [ .library( name: "TiktokenSwift", targets: ["TiktokenSwift"]), ], + dependencies: [ + .package(url: "https://github.com/DePasqualeOrg/swift-tiktoken.git", from: "0.0.1") + ], targets: [ .target( name: "TiktokenSwift", - dependencies: ["TiktokenFFI"], + dependencies: [ + .product(name: "SwiftTiktoken", package: "swift-tiktoken") + ], path: "Sources/TiktokenSwift" ), - .binaryTarget( - name: "TiktokenFFI", - path: "Sources/TiktokenFFI/TiktokenFFI.xcframework" - ), .testTarget( name: "TiktokenSwiftTests", dependencies: ["TiktokenSwift"], path: "Tests/TiktokenSwiftTests" ), ] -) \ No newline at end of file +) diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/Info.plist b/Sources/TiktokenFFI/TiktokenFFI.xcframework/Info.plist deleted file mode 100644 index bf1a2c1..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/Info.plist +++ /dev/null @@ -1,59 +0,0 @@ - - - - - AvailableLibraries - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - ios-arm64_x86_64-simulator - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - x86_64 - - SupportedPlatform - ios - SupportedPlatformVariant - simulator - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - ios-arm64 - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - - SupportedPlatform - ios - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - macos-arm64_x86_64 - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - x86_64 - - SupportedPlatform - macos - - - CFBundlePackageType - XFWK - XCFrameworkFormatVersion - 1.0 - - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist deleted file mode 100644 index f51d88b..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - iPhoneOS - - CFBundleVersion - 1 - MinimumOSVersion - 13.0 - - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index aa1fa1c..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f458581ebf9ae3efb39c6ab7b3739f3abf6d0c4ab45324112c38994d05f06610 -size 34355456 diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist deleted file mode 100644 index c75dd01..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - iPhoneSimulator - - CFBundleVersion - 1 - MinimumOSVersion - 13.0 - - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI b/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index 1dda786..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5d50a9237488aef981712d5ec078f5ac5da2c15f354136eb256cd875e368065d -size 67910472 diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h b/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist b/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist deleted file mode 100644 index 56e4e40..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - MacOSX - - CFBundleVersion - 1 - MinimumOSVersion - 10.15 - - diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap b/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI b/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index 88f6525..0000000 --- a/Sources/TiktokenFFI/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:828f0e47180a8374e813c31dbcab1a8ce098a011f81f193abfec1ee24f574904 -size 68594952 diff --git a/Sources/TiktokenFFI/include/TiktokenFFI.h b/Sources/TiktokenFFI/include/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/Sources/TiktokenFFI/include/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/Sources/TiktokenFFI/lib/libtiktoken.a b/Sources/TiktokenFFI/lib/libtiktoken.a deleted file mode 100644 index c144fc6..0000000 --- a/Sources/TiktokenFFI/lib/libtiktoken.a +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3303dc7a8bfdb4ee9346d8dbea38b5c8204202084e277d86570b71c431210137 -size 35276088 diff --git a/Sources/TiktokenSwift/Cl100kBaseEncoderLoader.swift b/Sources/TiktokenSwift/Cl100kBaseEncoderLoader.swift deleted file mode 100644 index 8f28b08..0000000 --- a/Sources/TiktokenSwift/Cl100kBaseEncoderLoader.swift +++ /dev/null @@ -1,312 +0,0 @@ -import Foundation -#if canImport(CryptoKit) -import CryptoKit -#endif - -/// Loader for OpenAI encodings that downloads and caches vocabulary data (matching Python implementation) -public struct Cl100kBaseEncoderLoader { - - /// URLs for different encodings (matching Python implementation) - private static let encodingURLs = [ - "cl100k_base": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", - "r50k_base": "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken", - "p50k_base": "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken", - "o200k_base": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" - ] - - /// Expected hashes for verification (matching Python implementation) - private static let expectedHashes = [ - "cl100k_base": "223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7", - "r50k_base": "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930", - "p50k_base": "94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069", - "o200k_base": "446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d" - ] - - /// Special tokens for different encodings - private static let specialTokens: [String: [String: UInt32]] = [ - "cl100k_base": [ - "<|endoftext|>": 100257, - "<|fim_prefix|>": 100258, - "<|fim_middle|>": 100259, - "<|fim_suffix|>": 100260, - "<|endofprompt|>": 100276 - ], - "r50k_base": [ - "<|endoftext|>": 50256 - ], - "p50k_base": [ - "<|endoftext|>": 50256 - ], - "o200k_base": [ - "<|endoftext|>": 199999, - "<|endofprompt|>": 200018 - ] - // o200k_harmony special tokens are handled in loadO200kHarmony() - ] - - /// Patterns for different encodings - private static let patterns: [String: String] = [ - "cl100k_base": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}++|\\p{N}{1,3}+| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*+|\\s++$|\\s*[\\r\\n]|\\s+(?!\\S)|\\s", - "r50k_base": "'(?:[sdmt]|ll|ve|re)| ?\\p{L}++| ?\\p{N}++| ?[^\\s\\p{L}\\p{N}]++|\\s++$|\\s+(?!\\S)|\\s", - "p50k_base": "'(?:[sdmt]|ll|ve|re)| ?\\p{L}++| ?\\p{N}++| ?[^\\s\\p{L}\\p{N}]++|\\s++$|\\s+(?!\\S)|\\s", - "o200k_base": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" - ] - - /// Cache directory for storing downloaded vocabularies - private static var cacheDirectory: URL { - let documentsPath = FileManager.default.urls(for: .cachesDirectory, - in: .userDomainMask).first! - return documentsPath.appendingPathComponent("tiktoken", isDirectory: true) - } - - /// Load an encoder for the specified encoding name - public static func loadEncoder(named encodingName: String) async throws -> CoreBpe { - // Special handling for o200k_harmony (based on o200k_base) - if encodingName == "o200k_harmony" { - return try await loadO200kHarmony() - } - - // Check if we have a cached version - let cacheURL = cacheDirectory.appendingPathComponent("\(encodingName).tiktoken") - - if FileManager.default.fileExists(atPath: cacheURL.path) { - print("📂 Loading cached \(encodingName) from: \(cacheURL.path)") - return try await loadFromFile(cacheURL, encodingName: encodingName) - } - - // Download if not cached - guard let urlString = encodingURLs[encodingName], - let url = URL(string: urlString) else { - throw LoadError.unsupportedEncoding(encodingName) - } - - print("⬇️ Downloading \(encodingName) from: \(url)") - - // Create cache directory if needed - try FileManager.default.createDirectory(at: cacheDirectory, - withIntermediateDirectories: true) - - // Download the data - let (data, _) = try await URLSession.shared.data(from: url) - - // Verify hash if available - #if canImport(CryptoKit) - if let expectedHash = expectedHashes[encodingName] { - let hash = SHA256.hash(data: data) - let hashString = hash.compactMap { String(format: "%02x", $0) }.joined() - - if hashString != expectedHash { - throw LoadError.hashMismatch(expected: expectedHash, actual: hashString) - } - print("✅ Hash verified") - } - #endif - - // Save to cache - try data.write(to: cacheURL) - print("💾 Saved to cache: \(cacheURL.path)") - - // Load and return - return try await loadFromData(data, encodingName: encodingName) - } - - /// Load encoder from cached file - private static func loadFromFile(_ url: URL, encodingName: String) async throws -> CoreBpe { - let data = try Data(contentsOf: url) - return try await loadFromData(data, encodingName: encodingName) - } - - /// Load encoder from data - private static func loadFromData(_ data: Data, encodingName: String) async throws -> CoreBpe { - // Parse the tiktoken format (which is a custom binary format) - let mergeableRanks = try parseTiktokenBpe(data) - - // Get special tokens and pattern for this encoding - let specialTokens = self.specialTokens[encodingName] ?? [:] - let pattern = patterns[encodingName] ?? patterns["cl100k_base"]! - - print("📊 Loaded \(encodingName):") - print(" Vocabulary size: \(mergeableRanks.count)") - print(" Special tokens: \(specialTokens.count)") - - // Create the encoder - return try newCoreBpe( - encoder: mergeableRanks, - specialTokensEncoder: specialTokens, - pattern: pattern - ) - } - - /// Load o200k_harmony encoding (based on o200k_base with additional special tokens) - private static func loadO200kHarmony() async throws -> CoreBpe { - // First load o200k_base - _ = try await loadEncoder(named: "o200k_base") - - // o200k_harmony uses the same base vocabulary and pattern as o200k_base - // but has many additional special tokens - var harmonySpecialTokens: [String: UInt32] = [ - "<|startoftext|>": 199998, - "<|endoftext|>": 199999, - "<|reserved_200000|>": 200000, - "<|reserved_200001|>": 200001, - "<|return|>": 200002, - "<|constrain|>": 200003, - "<|reserved_200004|>": 200004, - "<|channel|>": 200005, - "<|start|>": 200006, - "<|end|>": 200007, - "<|message|>": 200008, - "<|reserved_200009|>": 200009, - "<|reserved_200010|>": 200010, - "<|reserved_200011|>": 200011, - "<|call|>": 200012, - "<|endofprompt|>": 200018 - ] - - // Add reserved tokens from 200013 to 201087 - for i in 200013...201087 { - harmonySpecialTokens["<|reserved_\(i)|>"] = UInt32(i) - } - - print("📊 Loaded o200k_harmony:") - print(" Based on o200k_base vocabulary") - print(" Special tokens: \(harmonySpecialTokens.count)") - - // We need to reconstruct the encoder with the new special tokens - // Since we can't access the internal encoder data from CoreBpe, - // we'll need to load the o200k_base data directly - let cacheURL = cacheDirectory.appendingPathComponent("o200k_base.tiktoken") - - // Make sure o200k_base is downloaded - if !FileManager.default.fileExists(atPath: cacheURL.path) { - // This will download and cache o200k_base - _ = try await loadEncoder(named: "o200k_base") - } - - // Now load the raw data and create o200k_harmony - let data = try Data(contentsOf: cacheURL) - let mergeableRanks = try parseTiktokenBpe(data) - let pattern = patterns["o200k_base"] ?? patterns["cl100k_base"]! - - return try newCoreBpe( - encoder: mergeableRanks, - specialTokensEncoder: harmonySpecialTokens, - pattern: pattern - ) - } - - /// Parse tiktoken BPE format - /// The format is: base64-encoded token followed by space and rank - private static func parseTiktokenBpe(_ data: Data) throws -> [[UInt8]: UInt32] { - guard let content = String(data: data, encoding: .utf8) else { - throw LoadError.invalidData - } - - var encoder: [[UInt8]: UInt32] = [:] - - // Split by lines and parse each line - let lines = content.split(separator: "\n") - for line in lines { - let trimmed = line.trimmingCharacters(in: .whitespaces) - if trimmed.isEmpty { continue } - - // Each line has format: "base64_token rank" - let parts = trimmed.split(separator: " ", maxSplits: 1) - guard parts.count == 2, - let rank = UInt32(parts[1]) else { - continue - } - - // Decode the base64 token - guard let tokenData = Data(base64Encoded: String(parts[0])) else { - continue - } - - // Store as byte array directly - no need for base64 encoding - encoder[Array(tokenData)] = rank - } - - return encoder - } - - /// Clear the cache directory - public static func clearCache() throws { - if FileManager.default.fileExists(atPath: cacheDirectory.path) { - try FileManager.default.removeItem(at: cacheDirectory) - } - } - - /// Get cache size in bytes - public static func cacheSize() -> Int64 { - guard let enumerator = FileManager.default.enumerator( - at: cacheDirectory, - includingPropertiesForKeys: [.fileSizeKey], - options: [.skipsHiddenFiles] - ) else { - return 0 - } - - var totalSize: Int64 = 0 - for case let fileURL as URL in enumerator { - if let fileSize = try? fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize { - totalSize += Int64(fileSize) - } - } - - return totalSize - } - - /// Errors that can occur during loading - public enum LoadError: LocalizedError { - case unsupportedEncoding(String) - case downloadFailed(Error) - case invalidData - case hashMismatch(expected: String, actual: String) - - public var errorDescription: String? { - switch self { - case .unsupportedEncoding(let name): - return "Unsupported encoding: \(name)" - case .downloadFailed(let error): - return "Download failed: \(error.localizedDescription)" - case .invalidData: - return "Invalid tiktoken data format" - case .hashMismatch(let expected, let actual): - return "Hash mismatch - expected: \(expected), actual: \(actual)" - } - } - } -} - -// Convenience extension -public extension CoreBpe { - /// Load a standard OpenAI encoding by name - static func loadEncoding(named name: String) async throws -> CoreBpe { - return try await Cl100kBaseEncoderLoader.loadEncoder(named: name) - } - - /// Load cl100k_base encoding (most common) - static func cl100kBase() async throws -> CoreBpe { - return try await loadEncoding(named: "cl100k_base") - } - - /// Load r50k_base encoding - static func r50kBase() async throws -> CoreBpe { - return try await loadEncoding(named: "r50k_base") - } - - /// Load p50k_base encoding - static func p50kBase() async throws -> CoreBpe { - return try await loadEncoding(named: "p50k_base") - } - - /// Load o200k_base encoding - static func o200kBase() async throws -> CoreBpe { - return try await loadEncoding(named: "o200k_base") - } - - /// Load o200k_harmony encoding (based on o200k_base with additional special tokens) - static func o200kHarmony() async throws -> CoreBpe { - return try await loadEncoding(named: "o200k_harmony") - } -} \ No newline at end of file diff --git a/Sources/TiktokenSwift/Cl100kBaseLoader.swift b/Sources/TiktokenSwift/Cl100kBaseLoader.swift deleted file mode 100644 index 1148bc3..0000000 --- a/Sources/TiktokenSwift/Cl100kBaseLoader.swift +++ /dev/null @@ -1,90 +0,0 @@ -import Foundation - -/// Loader for cl100k_base encoding data -public struct Cl100kBaseLoader { - - /// Encoding data format for cl100k_base - public struct EncodingData: Codable { - let name: String - let vocabSize: UInt32 - let maxTokenValue: UInt32 - let pattern: String - let specialTokens: [String: UInt32] - let tokenBytes: [String?] // Base64 encoded bytes, indexed by token ID - let format: String - - enum CodingKeys: String, CodingKey { - case name - case vocabSize = "vocab_size" - case maxTokenValue = "max_token_value" - case pattern - case specialTokens = "special_tokens" - case tokenBytes = "token_bytes" - case format - } - } - - /// Load cl100k_base encoding from a JSON file - public static func loadFromFile(_ path: String) throws -> CoreBpe { - let url = URL(fileURLWithPath: path) - let data = try Data(contentsOf: url) - let encodingData = try JSONDecoder().decode(EncodingData.self, from: data) - - return try createEncoder(from: encodingData) - } - - /// Load cl100k_base encoding from embedded data - public static func loadFromData(_ data: Data) throws -> CoreBpe { - let encodingData = try JSONDecoder().decode(EncodingData.self, from: data) - return try createEncoder(from: encodingData) - } - - /// Create encoder from encoding data - private static func createEncoder(from encodingData: EncodingData) throws -> CoreBpe { - print("📂 Loading \(encodingData.name) encoding...") - print(" Vocabulary size: \(encodingData.vocabSize)") - print(" Token bytes: \(encodingData.tokenBytes.compactMap { $0 }.count)") - - // Build the encoder dictionary from token bytes - var encoder: [[UInt8]: UInt32] = [:] - - // Process each token - for (tokenId, base64Bytes) in encodingData.tokenBytes.enumerated() { - guard let base64Bytes = base64Bytes, - let tokenData = Data(base64Encoded: base64Bytes) else { - continue - } - - // Store as byte array directly - encoder[Array(tokenData)] = UInt32(tokenId) - } - - print(" Built encoder with \(encoder.count) entries") - - // Create the CoreBpe instance - return try newCoreBpe( - encoder: encoder, - specialTokensEncoder: encodingData.specialTokens, - pattern: encodingData.pattern - ) - } - - /// Create a proper BPE encoder that matches tiktoken's behavior - /// Note: This is a simplified version. Full BPE implementation would need: - /// 1. Proper byte-level BPE tokenization - /// 2. Merge rules application - /// 3. Exact pattern matching as tiktoken - public static func createCl100kBaseEncoder(from data: EncodingData) throws -> CoreBpe { - // This would need a more sophisticated implementation to exactly match tiktoken - // For now, we'll use the simple token-to-string mapping - return try createEncoder(from: data) - } -} - -// Convenience extension -public extension CoreBpe { - /// Load cl100k_base encoding from a file - static func cl100kBase(fromFile path: String) throws -> CoreBpe { - return try Cl100kBaseLoader.loadFromFile(path) - } -} \ No newline at end of file diff --git a/Sources/TiktokenSwift/EncodingLoader.swift b/Sources/TiktokenSwift/EncodingLoader.swift deleted file mode 100644 index 98c9985..0000000 --- a/Sources/TiktokenSwift/EncodingLoader.swift +++ /dev/null @@ -1,104 +0,0 @@ -import Foundation - -/// Utilities for loading tiktoken encodings -public struct EncodingLoader { - - /// Errors that can occur during encoding loading - public enum LoadError: Error { - case fileNotFound(String) - case invalidFormat - case decodingError(Error) - } - - /// Encoding data structure matching Python's format - public struct EncodingData: Codable { - let name: String - let explicitNVocab: UInt32 - let maxTokenValue: UInt32 - let specialTokens: [String: UInt32] - let testVocabulary: [String: UInt32]? - let pattern: String - - enum CodingKeys: String, CodingKey { - case name - case explicitNVocab = "explicit_n_vocab" - case maxTokenValue = "max_token_value" - case specialTokens = "special_tokens" - case testVocabulary = "test_vocabulary" - case pattern - } - } - - /// Load encoding data from a JSON file - public static func loadEncoding(from path: String) throws -> CoreBpe { - let url = URL(fileURLWithPath: path) - - guard FileManager.default.fileExists(atPath: path) else { - throw LoadError.fileNotFound(path) - } - - do { - let data = try Data(contentsOf: url) - let encodingData = try JSONDecoder().decode(EncodingData.self, from: data) - - // For now, use test vocabulary if available - // In production, you'd load the full BPE ranks - let stringEncoder = encodingData.testVocabulary ?? [:] - - // Convert string keys to byte arrays - var encoder: [[UInt8]: UInt32] = [:] - for (key, value) in stringEncoder { - encoder[Array(key.utf8)] = value - } - - return try newCoreBpe( - encoder: encoder, - specialTokensEncoder: encodingData.specialTokens, - pattern: encodingData.pattern - ) - - } catch let error as DecodingError { - throw LoadError.decodingError(error) - } catch { - throw error - } - } - - /// Create a cl100k_base encoder from embedded data - /// Note: In production, this would load the actual cl100k_base data - public static func createCl100kBase() throws -> CoreBpe { - // This is a placeholder - in reality, you'd embed the actual - // cl100k_base data or load it from a resource - print("⚠️ Warning: Using test encoder. Load actual cl100k_base data for production use.") - return try TiktokenHelper.createTestEncoder() - } - - /// Load encoding from embedded resources - public static func loadEmbeddedEncoding(named name: String) throws -> CoreBpe { - // In a real implementation, you'd load from bundle resources - // For example: - // guard let url = Bundle.module.url(forResource: name, withExtension: "json") else { - // throw LoadError.fileNotFound(name) - // } - - switch name { - case "cl100k_base": - return try createCl100kBase() - default: - throw LoadError.fileNotFound("Encoding '\(name)' not found") - } - } -} - -// Extension to make loading easier -public extension CoreBpe { - /// Load a named encoding - static func encoding(named name: String) throws -> CoreBpe { - return try EncodingLoader.loadEmbeddedEncoding(named: name) - } - - /// Load encoding from a JSON file - static func encoding(fromFile path: String) throws -> CoreBpe { - return try EncodingLoader.loadEncoding(from: path) - } -} \ No newline at end of file diff --git a/Sources/TiktokenSwift/TiktokenFFI.swift b/Sources/TiktokenSwift/TiktokenFFI.swift deleted file mode 100644 index 4cb64f4..0000000 --- a/Sources/TiktokenSwift/TiktokenFFI.swift +++ /dev/null @@ -1,860 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -// swiftlint:disable all -import Foundation -import TiktokenFFI -// Depending on the consumer's build setup, the low-level FFI code -// might be in a separate module, or it might be compiled inline into -// this module. This is a bit of light hackery to work with both. - -fileprivate extension RustBuffer { - // Allocate a new buffer, copying the contents of a `UInt8` array. - init(bytes: [UInt8]) { - let rbuf = bytes.withUnsafeBufferPointer { ptr in - RustBuffer.from(ptr) - } - self.init(capacity: rbuf.capacity, len: rbuf.len, data: rbuf.data) - } - - static func empty() -> RustBuffer { - RustBuffer(capacity: 0, len:0, data: nil) - } - - static func from(_ ptr: UnsafeBufferPointer) -> RustBuffer { - try! rustCall { ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes(bufferPointer: ptr), $0) } - } - - // Frees the buffer in place. - // The buffer must not be used after this is called. - func deallocate() { - try! rustCall { ffi_tiktoken_rustbuffer_free(self, $0) } - } -} - -fileprivate extension ForeignBytes { - init(bufferPointer: UnsafeBufferPointer) { - self.init(len: Int32(bufferPointer.count), data: bufferPointer.baseAddress) - } -} - -// For every type used in the interface, we provide helper methods for conveniently -// lifting and lowering that type from C-compatible data, and for reading and writing -// values of that type in a buffer. - -// Helper classes/extensions that don't change. -// Someday, this will be in a library of its own. - -fileprivate extension Data { - init(rustBuffer: RustBuffer) { - self.init( - bytesNoCopy: rustBuffer.data!, - count: Int(rustBuffer.len), - deallocator: .none - ) - } -} - -// Define reader functionality. Normally this would be defined in a class or -// struct, but we use standalone functions instead in order to make external -// types work. -// -// With external types, one swift source file needs to be able to call the read -// method on another source file's FfiConverter, but then what visibility -// should Reader have? -// - If Reader is fileprivate, then this means the read() must also -// be fileprivate, which doesn't work with external types. -// - If Reader is internal/public, we'll get compile errors since both source -// files will try define the same type. -// -// Instead, the read() method and these helper functions input a tuple of data - -fileprivate func createReader(data: Data) -> (data: Data, offset: Data.Index) { - (data: data, offset: 0) -} - -// Reads an integer at the current offset, in big-endian order, and advances -// the offset on success. Throws if reading the integer would move the -// offset past the end of the buffer. -fileprivate func readInt(_ reader: inout (data: Data, offset: Data.Index)) throws -> T { - let range = reader.offset...size - guard reader.data.count >= range.upperBound else { - throw UniffiInternalError.bufferOverflow - } - if T.self == UInt8.self { - let value = reader.data[reader.offset] - reader.offset += 1 - return value as! T - } - var value: T = 0 - let _ = withUnsafeMutableBytes(of: &value, { reader.data.copyBytes(to: $0, from: range)}) - reader.offset = range.upperBound - return value.bigEndian -} - -// Reads an arbitrary number of bytes, to be used to read -// raw bytes, this is useful when lifting strings -fileprivate func readBytes(_ reader: inout (data: Data, offset: Data.Index), count: Int) throws -> Array { - let range = reader.offset..<(reader.offset+count) - guard reader.data.count >= range.upperBound else { - throw UniffiInternalError.bufferOverflow - } - var value = [UInt8](repeating: 0, count: count) - value.withUnsafeMutableBufferPointer({ buffer in - reader.data.copyBytes(to: buffer, from: range) - }) - reader.offset = range.upperBound - return value -} - -// Reads a float at the current offset. -fileprivate func readFloat(_ reader: inout (data: Data, offset: Data.Index)) throws -> Float { - return Float(bitPattern: try readInt(&reader)) -} - -// Reads a float at the current offset. -fileprivate func readDouble(_ reader: inout (data: Data, offset: Data.Index)) throws -> Double { - return Double(bitPattern: try readInt(&reader)) -} - -// Indicates if the offset has reached the end of the buffer. -fileprivate func hasRemaining(_ reader: (data: Data, offset: Data.Index)) -> Bool { - return reader.offset < reader.data.count -} - -// Define writer functionality. Normally this would be defined in a class or -// struct, but we use standalone functions instead in order to make external -// types work. See the above discussion on Readers for details. - -fileprivate func createWriter() -> [UInt8] { - return [] -} - -fileprivate func writeBytes(_ writer: inout [UInt8], _ byteArr: S) where S: Sequence, S.Element == UInt8 { - writer.append(contentsOf: byteArr) -} - -// Writes an integer in big-endian order. -// -// Warning: make sure what you are trying to write -// is in the correct type! -fileprivate func writeInt(_ writer: inout [UInt8], _ value: T) { - var value = value.bigEndian - withUnsafeBytes(of: &value) { writer.append(contentsOf: $0) } -} - -fileprivate func writeFloat(_ writer: inout [UInt8], _ value: Float) { - writeInt(&writer, value.bitPattern) -} - -fileprivate func writeDouble(_ writer: inout [UInt8], _ value: Double) { - writeInt(&writer, value.bitPattern) -} - -// Protocol for types that transfer other types across the FFI. This is -// analogous to the Rust trait of the same name. -fileprivate protocol FfiConverter { - associatedtype FfiType - associatedtype SwiftType - - static func lift(_ value: FfiType) throws -> SwiftType - static func lower(_ value: SwiftType) -> FfiType - static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType - static func write(_ value: SwiftType, into buf: inout [UInt8]) -} - -// Types conforming to `Primitive` pass themselves directly over the FFI. -fileprivate protocol FfiConverterPrimitive: FfiConverter where FfiType == SwiftType { } - -extension FfiConverterPrimitive { -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public static func lift(_ value: FfiType) throws -> SwiftType { - return value - } - -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public static func lower(_ value: SwiftType) -> FfiType { - return value - } -} - -// Types conforming to `FfiConverterRustBuffer` lift and lower into a `RustBuffer`. -// Used for complex types where it's hard to write a custom lift/lower. -fileprivate protocol FfiConverterRustBuffer: FfiConverter where FfiType == RustBuffer {} - -extension FfiConverterRustBuffer { -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public static func lift(_ buf: RustBuffer) throws -> SwiftType { - var reader = createReader(data: Data(rustBuffer: buf)) - let value = try read(from: &reader) - if hasRemaining(reader) { - throw UniffiInternalError.incompleteData - } - buf.deallocate() - return value - } - -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public static func lower(_ value: SwiftType) -> RustBuffer { - var writer = createWriter() - write(value, into: &writer) - return RustBuffer(bytes: writer) - } -} -// An error type for FFI errors. These errors occur at the UniFFI level, not -// the library level. -fileprivate enum UniffiInternalError: LocalizedError { - case bufferOverflow - case incompleteData - case unexpectedOptionalTag - case unexpectedEnumCase - case unexpectedNullPointer - case unexpectedRustCallStatusCode - case unexpectedRustCallError - case unexpectedStaleHandle - case rustPanic(_ message: String) - - public var errorDescription: String? { - switch self { - case .bufferOverflow: return "Reading the requested value would read past the end of the buffer" - case .incompleteData: return "The buffer still has data after lifting its containing value" - case .unexpectedOptionalTag: return "Unexpected optional tag; should be 0 or 1" - case .unexpectedEnumCase: return "Raw enum value doesn't match any cases" - case .unexpectedNullPointer: return "Raw pointer value was null" - case .unexpectedRustCallStatusCode: return "Unexpected RustCallStatus code" - case .unexpectedRustCallError: return "CALL_ERROR but no errorClass specified" - case .unexpectedStaleHandle: return "The object in the handle map has been dropped already" - case let .rustPanic(message): return message - } - } -} - -fileprivate extension NSLock { - func withLock(f: () throws -> T) rethrows -> T { - self.lock() - defer { self.unlock() } - return try f() - } -} - -fileprivate let CALL_SUCCESS: Int8 = 0 -fileprivate let CALL_ERROR: Int8 = 1 -fileprivate let CALL_UNEXPECTED_ERROR: Int8 = 2 -fileprivate let CALL_CANCELLED: Int8 = 3 - -fileprivate extension RustCallStatus { - init() { - self.init( - code: CALL_SUCCESS, - errorBuf: RustBuffer.init( - capacity: 0, - len: 0, - data: nil - ) - ) - } -} - -private func rustCall(_ callback: (UnsafeMutablePointer) -> T) throws -> T { - let neverThrow: ((RustBuffer) throws -> Never)? = nil - return try makeRustCall(callback, errorHandler: neverThrow) -} - -private func rustCallWithError( - _ errorHandler: @escaping (RustBuffer) throws -> E, - _ callback: (UnsafeMutablePointer) -> T) throws -> T { - try makeRustCall(callback, errorHandler: errorHandler) -} - -private func makeRustCall( - _ callback: (UnsafeMutablePointer) -> T, - errorHandler: ((RustBuffer) throws -> E)? -) throws -> T { - var callStatus = RustCallStatus.init() - let returnedVal = callback(&callStatus) - try uniffiCheckCallStatus(callStatus: callStatus, errorHandler: errorHandler) - return returnedVal -} - -private func uniffiCheckCallStatus( - callStatus: RustCallStatus, - errorHandler: ((RustBuffer) throws -> E)? -) throws { - switch callStatus.code { - case CALL_SUCCESS: - return - - case CALL_ERROR: - if let errorHandler = errorHandler { - throw try errorHandler(callStatus.errorBuf) - } else { - callStatus.errorBuf.deallocate() - throw UniffiInternalError.unexpectedRustCallError - } - - case CALL_UNEXPECTED_ERROR: - // When the rust code sees a panic, it tries to construct a RustBuffer - // with the message. But if that code panics, then it just sends back - // an empty buffer. - if callStatus.errorBuf.len > 0 { - throw UniffiInternalError.rustPanic(try FfiConverterString.lift(callStatus.errorBuf)) - } else { - callStatus.errorBuf.deallocate() - throw UniffiInternalError.rustPanic("Rust panic") - } - - case CALL_CANCELLED: - fatalError("Cancellation not supported yet") - - default: - throw UniffiInternalError.unexpectedRustCallStatusCode - } -} - -private func uniffiTraitInterfaceCall( - callStatus: UnsafeMutablePointer, - makeCall: () throws -> T, - writeReturn: (T) -> () -) { - do { - try writeReturn(makeCall()) - } catch let error { - callStatus.pointee.code = CALL_UNEXPECTED_ERROR - callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) - } -} - -private func uniffiTraitInterfaceCallWithError( - callStatus: UnsafeMutablePointer, - makeCall: () throws -> T, - writeReturn: (T) -> (), - lowerError: (E) -> RustBuffer -) { - do { - try writeReturn(makeCall()) - } catch let error as E { - callStatus.pointee.code = CALL_ERROR - callStatus.pointee.errorBuf = lowerError(error) - } catch { - callStatus.pointee.code = CALL_UNEXPECTED_ERROR - callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) - } -} -// Initial value and increment amount for handles. -// These ensure that SWIFT handles always have the lowest bit set -fileprivate let UNIFFI_HANDLEMAP_INITIAL: UInt64 = 1 -fileprivate let UNIFFI_HANDLEMAP_DELTA: UInt64 = 2 - -fileprivate final class UniffiHandleMap: @unchecked Sendable { - // All mutation happens with this lock held, which is why we implement @unchecked Sendable. - private let lock = NSLock() - private var map: [UInt64: T] = [:] - private var currentHandle: UInt64 = UNIFFI_HANDLEMAP_INITIAL - - func insert(obj: T) -> UInt64 { - lock.withLock { - return doInsert(obj) - } - } - - // Low-level insert function, this assumes `lock` is held. - private func doInsert(_ obj: T) -> UInt64 { - let handle = currentHandle - currentHandle += UNIFFI_HANDLEMAP_DELTA - map[handle] = obj - return handle - } - - func get(handle: UInt64) throws -> T { - try lock.withLock { - guard let obj = map[handle] else { - throw UniffiInternalError.unexpectedStaleHandle - } - return obj - } - } - - func clone(handle: UInt64) throws -> UInt64 { - try lock.withLock { - guard let obj = map[handle] else { - throw UniffiInternalError.unexpectedStaleHandle - } - return doInsert(obj) - } - } - - @discardableResult - func remove(handle: UInt64) throws -> T { - try lock.withLock { - guard let obj = map.removeValue(forKey: handle) else { - throw UniffiInternalError.unexpectedStaleHandle - } - return obj - } - } - - var count: Int { - get { - map.count - } - } -} - - -// Public interface members begin here. - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterUInt8: FfiConverterPrimitive { - typealias FfiType = UInt8 - typealias SwiftType = UInt8 - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt8 { - return try lift(readInt(&buf)) - } - - public static func write(_ value: UInt8, into buf: inout [UInt8]) { - writeInt(&buf, lower(value)) - } -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterUInt32: FfiConverterPrimitive { - typealias FfiType = UInt32 - typealias SwiftType = UInt32 - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt32 { - return try lift(readInt(&buf)) - } - - public static func write(_ value: SwiftType, into buf: inout [UInt8]) { - writeInt(&buf, lower(value)) - } -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterString: FfiConverter { - typealias SwiftType = String - typealias FfiType = RustBuffer - - public static func lift(_ value: RustBuffer) throws -> String { - defer { - value.deallocate() - } - if value.data == nil { - return String() - } - let bytes = UnsafeBufferPointer(start: value.data!, count: Int(value.len)) - return String(bytes: bytes, encoding: String.Encoding.utf8)! - } - - public static func lower(_ value: String) -> RustBuffer { - return value.utf8CString.withUnsafeBufferPointer { ptr in - // The swift string gives us int8_t, we want uint8_t. - ptr.withMemoryRebound(to: UInt8.self) { ptr in - // The swift string gives us a trailing null byte, we don't want it. - let buf = UnsafeBufferPointer(rebasing: ptr.prefix(upTo: ptr.count - 1)) - return RustBuffer.from(buf) - } - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> String { - let len: Int32 = try readInt(&buf) - return String(bytes: try readBytes(&buf, count: Int(len)), encoding: String.Encoding.utf8)! - } - - public static func write(_ value: String, into buf: inout [UInt8]) { - let len = Int32(value.utf8.count) - writeInt(&buf, len) - writeBytes(&buf, value.utf8) - } -} - - - - -public protocol CoreBpeProtocol: AnyObject, Sendable { - - func decodeBytes(tokens: [UInt32]) throws -> [UInt8] - - func decode(tokens: [UInt32]) throws -> String? - - func encode(text: String, allowedSpecial: [String]) -> [UInt32] - - func encodeOrdinary(text: String) -> [UInt32] - - func encodeWithSpecialTokens(text: String) -> [UInt32] - -} -open class CoreBpe: CoreBpeProtocol, @unchecked Sendable { - fileprivate let handle: UInt64 - - /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public struct NoHandle { - public init() {} - } - - // TODO: We'd like this to be `private` but for Swifty reasons, - // we can't implement `FfiConverter` without making this `required` and we can't - // make it `required` without making it `public`. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - required public init(unsafeFromHandle handle: UInt64) { - self.handle = handle - } - - // This constructor can be used to instantiate a fake object. - // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. - // - // - Warning: - // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash. -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public init(noHandle: NoHandle) { - self.handle = 0 - } - -#if swift(>=5.8) - @_documentation(visibility: private) -#endif - public func uniffiCloneHandle() -> UInt64 { - return try! rustCall { uniffi_tiktoken_fn_clone_corebpe(self.handle, $0) } - } - // No primary constructor declared for this class. - - deinit { - try! rustCall { uniffi_tiktoken_fn_free_corebpe(handle, $0) } - } - - - - -open func decodeBytes(tokens: [UInt32])throws -> [UInt8] { - return try FfiConverterSequenceUInt8.lift(try rustCallWithError(FfiConverterTypeTiktokenError_lift) { - uniffi_tiktoken_fn_method_corebpe_decode_bytes(self.uniffiCloneHandle(), - FfiConverterSequenceUInt32.lower(tokens),$0 - ) -}) -} - -open func decode(tokens: [UInt32]) throws -> String? { - let bytes = try decodeBytes(tokens: tokens) - return String(bytes: bytes, encoding: .utf8) -} - -open func encode(text: String, allowedSpecial: [String]) -> [UInt32] { - return try! FfiConverterSequenceUInt32.lift(try! rustCall() { - uniffi_tiktoken_fn_method_corebpe_encode(self.uniffiCloneHandle(), - FfiConverterString.lower(text), - FfiConverterSequenceString.lower(allowedSpecial),$0 - ) -}) -} - -open func encodeOrdinary(text: String) -> [UInt32] { - return try! FfiConverterSequenceUInt32.lift(try! rustCall() { - uniffi_tiktoken_fn_method_corebpe_encode_ordinary(self.uniffiCloneHandle(), - FfiConverterString.lower(text),$0 - ) -}) -} - -open func encodeWithSpecialTokens(text: String) -> [UInt32] { - return try! FfiConverterSequenceUInt32.lift(try! rustCall() { - uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(self.uniffiCloneHandle(), - FfiConverterString.lower(text),$0 - ) -}) -} - - -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public struct FfiConverterTypeCoreBpe: FfiConverter { - typealias FfiType = UInt64 - typealias SwiftType = CoreBpe - - public static func lift(_ handle: UInt64) throws -> CoreBpe { - return CoreBpe(unsafeFromHandle: handle) - } - - public static func lower(_ value: CoreBpe) -> UInt64 { - return value.uniffiCloneHandle() - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> CoreBpe { - let handle: UInt64 = try readInt(&buf) - return try lift(handle) - } - - public static func write(_ value: CoreBpe, into buf: inout [UInt8]) { - writeInt(&buf, lower(value)) - } -} - - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeCoreBpe_lift(_ handle: UInt64) throws -> CoreBpe { - return try FfiConverterTypeCoreBpe.lift(handle) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeCoreBpe_lower(_ value: CoreBpe) -> UInt64 { - return FfiConverterTypeCoreBpe.lower(value) -} - - - - -public enum TiktokenError: Swift.Error { - - - - case RegexError(message: String - ) - case DecodeError(message: String - ) -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public struct FfiConverterTypeTiktokenError: FfiConverterRustBuffer { - typealias SwiftType = TiktokenError - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> TiktokenError { - let variant: Int32 = try readInt(&buf) - switch variant { - - - - - case 1: return .RegexError( - message: try FfiConverterString.read(from: &buf) - ) - case 2: return .DecodeError( - message: try FfiConverterString.read(from: &buf) - ) - - default: throw UniffiInternalError.unexpectedEnumCase - } - } - - public static func write(_ value: TiktokenError, into buf: inout [UInt8]) { - switch value { - - - - - - case let .RegexError(message): - writeInt(&buf, Int32(1)) - FfiConverterString.write(message, into: &buf) - - - case let .DecodeError(message): - writeInt(&buf, Int32(2)) - FfiConverterString.write(message, into: &buf) - - } - } -} - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeTiktokenError_lift(_ buf: RustBuffer) throws -> TiktokenError { - return try FfiConverterTypeTiktokenError.lift(buf) -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -public func FfiConverterTypeTiktokenError_lower(_ value: TiktokenError) -> RustBuffer { - return FfiConverterTypeTiktokenError.lower(value) -} - - -extension TiktokenError: Equatable, Hashable {} - - - - -extension TiktokenError: Foundation.LocalizedError { - public var errorDescription: String? { - String(reflecting: self) - } -} - - - - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterSequenceUInt8: FfiConverterRustBuffer { - typealias SwiftType = [UInt8] - - public static func write(_ value: [UInt8], into buf: inout [UInt8]) { - let len = Int32(value.count) - writeInt(&buf, len) - for item in value { - FfiConverterUInt8.write(item, into: &buf) - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [UInt8] { - let len: Int32 = try readInt(&buf) - var seq = [UInt8]() - seq.reserveCapacity(Int(len)) - for _ in 0 ..< len { - seq.append(try FfiConverterUInt8.read(from: &buf)) - } - return seq - } -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterSequenceUInt32: FfiConverterRustBuffer { - typealias SwiftType = [UInt32] - - public static func write(_ value: [UInt32], into buf: inout [UInt8]) { - let len = Int32(value.count) - writeInt(&buf, len) - for item in value { - FfiConverterUInt32.write(item, into: &buf) - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [UInt32] { - let len: Int32 = try readInt(&buf) - var seq = [UInt32]() - seq.reserveCapacity(Int(len)) - for _ in 0 ..< len { - seq.append(try FfiConverterUInt32.read(from: &buf)) - } - return seq - } -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterSequenceString: FfiConverterRustBuffer { - typealias SwiftType = [String] - - public static func write(_ value: [String], into buf: inout [UInt8]) { - let len = Int32(value.count) - writeInt(&buf, len) - for item in value { - FfiConverterString.write(item, into: &buf) - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [String] { - let len: Int32 = try readInt(&buf) - var seq = [String]() - seq.reserveCapacity(Int(len)) - for _ in 0 ..< len { - seq.append(try FfiConverterString.read(from: &buf)) - } - return seq - } -} - -#if swift(>=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterDictionaryStringUInt32: FfiConverterRustBuffer { - public static func write(_ value: [String: UInt32], into buf: inout [UInt8]) { - let len = Int32(value.count) - writeInt(&buf, len) - for (key, value) in value { - FfiConverterString.write(key, into: &buf) - FfiConverterUInt32.write(value, into: &buf) - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [String: UInt32] { - let len: Int32 = try readInt(&buf) - var dict = [String: UInt32]() - dict.reserveCapacity(Int(len)) - for _ in 0..=5.8) -@_documentation(visibility: private) -#endif -fileprivate struct FfiConverterDictionarySequenceUInt8UInt32: FfiConverterRustBuffer { - public static func write(_ value: [[UInt8]: UInt32], into buf: inout [UInt8]) { - let len = Int32(value.count) - writeInt(&buf, len) - for (key, value) in value { - FfiConverterSequenceUInt8.write(key, into: &buf) - FfiConverterUInt32.write(value, into: &buf) - } - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> [[UInt8]: UInt32] { - let len: Int32 = try readInt(&buf) - var dict = [[UInt8]: UInt32]() - dict.reserveCapacity(Int(len)) - for _ in 0.. CoreBpe { - return try FfiConverterTypeCoreBpe_lift(try rustCallWithError(FfiConverterTypeTiktokenError_lift) { - uniffi_tiktoken_fn_func_new_core_bpe( - FfiConverterDictionarySequenceUInt8UInt32.lower(encoder), - FfiConverterDictionaryStringUInt32.lower(specialTokensEncoder), - FfiConverterString.lower(pattern),$0 - ) -}) -} - - -// swiftlint:enable all \ No newline at end of file diff --git a/Sources/TiktokenSwift/TiktokenHelper.swift b/Sources/TiktokenSwift/TiktokenHelper.swift deleted file mode 100644 index f15c7e4..0000000 --- a/Sources/TiktokenSwift/TiktokenHelper.swift +++ /dev/null @@ -1,53 +0,0 @@ -import Foundation - -/// Helper utilities for using Tiktoken -public struct TiktokenHelper { - /// Creates a test encoder with a small vocabulary - /// - Note: This is for testing only. In production, load real encoding data. - public static func createTestEncoder() throws -> CoreBpe { - // Create a more comprehensive test vocabulary with individual characters - var encoder: [[UInt8]: UInt32] = [:] - - // Add basic ASCII characters as individual tokens - for i in 32...126 { // printable ASCII - let char = UInt8(i) - encoder[[char]] = UInt32(i) - } - - // Add some common multi-byte sequences - encoder[Array("hello".utf8)] = 200 - encoder[Array("world".utf8)] = 201 - encoder[Array("test".utf8)] = 202 - encoder[Array("swift".utf8)] = 203 - encoder[Array("tiktoken".utf8)] = 204 - encoder[Array(" ".utf8)] = 32 // space - encoder[Array("!".utf8)] = 33 // exclamation - - // Add newline and other control characters - encoder[Array("\n".utf8)] = 10 - encoder[Array("\r".utf8)] = 13 - encoder[Array("\t".utf8)] = 9 - - let specialTokens: [String: UInt32] = ["<|endoftext|>": 100257] - let pattern = "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}++|\\p{N}{1,3}+| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*+|\\s++$|\\s*[\\r\\n]|\\s+(?!\\S)|\\s" - - return try newCoreBpe( - encoder: encoder, - specialTokensEncoder: specialTokens, - pattern: pattern - ) - } -} - -// Extension for convenience methods -public extension CoreBpe { - /// Encodes text without special tokens - func encodeText(_ text: String) -> [UInt32] { - return encode(text: text, allowedSpecial: []) - } - - /// Decodes tokens to string - func decodeTokens(_ tokens: [UInt32]) -> String? { - return try? decode(tokens: tokens) - } -} diff --git a/Sources/TiktokenSwift/TiktokenSwift.swift b/Sources/TiktokenSwift/TiktokenSwift.swift new file mode 100644 index 0000000..30fa37a --- /dev/null +++ b/Sources/TiktokenSwift/TiktokenSwift.swift @@ -0,0 +1,3 @@ +@_exported import SwiftTiktoken + +public typealias CoreBpe = CoreBPE diff --git a/Tests/TiktokenSwiftTests/DownloadVocabularies.swift b/Tests/TiktokenSwiftTests/DownloadVocabularies.swift deleted file mode 100644 index 47ea5d6..0000000 --- a/Tests/TiktokenSwiftTests/DownloadVocabularies.swift +++ /dev/null @@ -1,86 +0,0 @@ -import Foundation -import XCTest -@testable import TiktokenSwift - -/// Helper to download and cache vocabularies for testing -/// Run this once to download all vocabularies to the cache -final class DownloadVocabularies: XCTestCase { - - /// Download all vocabularies to cache - /// This test will download and cache all encodings so subsequent tests can use them - func testDownloadAllVocabularies() async throws { - let encodings = [ - "cl100k_base", - "r50k_base", - "p50k_base", - "o200k_base", - "gpt2" // gpt2 uses r50k_base - ] - - for encodingName in encodings { - do { - print("📥 Downloading \(encodingName)...") - let encoder = try await CoreBpe.loadEncoding(named: encodingName == "gpt2" ? "r50k_base" : encodingName) - - // Test it works - let tokens = encoder.encode(text: "test", allowedSpecial: []) - XCTAssertFalse(tokens.isEmpty) - - print("✅ Successfully downloaded and cached \(encodingName)") - } catch { - print("❌ Failed to download \(encodingName): \(error)") - // Don't fail the test - just log the error - } - } - } - - /// Helper method to manually download cl100k_base - func testDownloadCl100kBase() async throws { - print("📥 Downloading cl100k_base vocabulary...") - - let url = URL(string: "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken")! - let (data, _) = try await URLSession.shared.data(from: url) - - // Save to cache directory - let cacheDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! - .appendingPathComponent("tiktoken", isDirectory: true) - - try FileManager.default.createDirectory(at: cacheDir, withIntermediateDirectories: true) - - let cacheFile = cacheDir.appendingPathComponent("cl100k_base.tiktoken") - try data.write(to: cacheFile) - - print("✅ Saved to: \(cacheFile.path)") - print("📏 File size: \(data.count) bytes") - - } - - /// Helper to download all vocabularies directly - func testDirectDownload() async throws { - let vocabularies = [ - ("cl100k_base", "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"), - ("r50k_base", "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken"), - ("p50k_base", "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken"), - ("o200k_base", "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken") - ] - - let cacheDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! - .appendingPathComponent("tiktoken", isDirectory: true) - - try FileManager.default.createDirectory(at: cacheDir, withIntermediateDirectories: true) - - for (name, urlString) in vocabularies { - print("📥 Downloading \(name)...") - - let url = URL(string: urlString)! - let (data, _) = try await URLSession.shared.data(from: url) - - let cacheFile = cacheDir.appendingPathComponent("\(name).tiktoken") - try data.write(to: cacheFile) - - print("✅ Saved \(name) (\(data.count) bytes) to: \(cacheFile.path)") - } - - print("\n✨ All vocabularies downloaded to: \(cacheDir.path)") - } -} diff --git a/Tests/TiktokenSwiftTests/TestEncodingHelper.swift b/Tests/TiktokenSwiftTests/TestEncodingHelper.swift deleted file mode 100644 index b2cbc6e..0000000 --- a/Tests/TiktokenSwiftTests/TestEncodingHelper.swift +++ /dev/null @@ -1,73 +0,0 @@ -import Foundation -import XCTest -@testable import TiktokenSwift - -/// Helper to download and cache encodings for tests -struct TestEncodingHelper { - - /// Pre-download and cache encodings for testing - /// This helps tests run consistently without network issues during test execution - static func setupEncodingsIfNeeded() async throws { - // Try to download and cache common encodings - // These will be cached in ~/Library/Caches/tiktoken/ - - let encodingsToCache = [ - "cl100k_base", - "gpt2", - "r50k_base", - "p50k_base" - ] - - for encoding in encodingsToCache { - do { - print("📥 Attempting to cache \(encoding)...") - _ = try await CoreBpe.loadEncoding(named: encoding) - print("✅ Successfully cached \(encoding)") - } catch { - print("⚠️ Could not cache \(encoding): \(error)") - // Continue with other encodings - } - } - } - - /// Load cl100k_base from a local test file (if we have one) - static func loadCl100kBaseFromTestData() throws -> CoreBpe? { - // Check if we have a local test data file - let bundle = Bundle(for: TiktokenSwiftTests.self) - - // Try to find a test data file in the test bundle - if let testDataURL = bundle.url(forResource: "cl100k_base", withExtension: "tiktoken") { - print("📂 Loading cl100k_base from test data: \(testDataURL.path)") - return try EncodingLoader.loadEncoding(from: testDataURL.path) - } - - return nil - } - - /// Try to get an encoding, falling back to test encoder if needed - static func getEncodingOrTestEncoder(named name: String) async throws -> CoreBpe { - do { - return try await CoreBpe.loadEncoding(named: name) - } catch { - print("⚠️ Could not load \(name), using test encoder instead") - return try TiktokenHelper.createTestEncoder() - } - } -} - -/// XCTest extension to set up encodings once per test session -extension XCTestCase { - - /// Call this in setUpWithError() to ensure encodings are cached - func setupEncodingsOnce() async throws { - // Use a static flag to only download once per test session - struct Static { - static var hasSetup = false - } - - guard !Static.hasSetup else { return } - Static.hasSetup = true - - try await TestEncodingHelper.setupEncodingsIfNeeded() - } -} \ No newline at end of file diff --git a/TiktokenFFI.xcframework/Info.plist b/TiktokenFFI.xcframework/Info.plist deleted file mode 100644 index 48ea9c5..0000000 --- a/TiktokenFFI.xcframework/Info.plist +++ /dev/null @@ -1,59 +0,0 @@ - - - - - AvailableLibraries - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - macos-arm64_x86_64 - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - x86_64 - - SupportedPlatform - macos - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - ios-arm64_x86_64-simulator - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - x86_64 - - SupportedPlatform - ios - SupportedPlatformVariant - simulator - - - BinaryPath - TiktokenFFI.framework/TiktokenFFI - LibraryIdentifier - ios-arm64 - LibraryPath - TiktokenFFI.framework - SupportedArchitectures - - arm64 - - SupportedPlatform - ios - - - CFBundlePackageType - XFWK - XCFrameworkFormatVersion - 1.0 - - diff --git a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h b/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist b/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist deleted file mode 100644 index f51d88b..0000000 --- a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - iPhoneOS - - CFBundleVersion - 1 - MinimumOSVersion - 13.0 - - diff --git a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap b/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI b/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index c289bce..0000000 --- a/TiktokenFFI.xcframework/ios-arm64/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fe88439c70cd58c64a50a8640efcc4f3128c5cbc3df3f551d00a51109ecf7255 -size 35098512 diff --git a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h b/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist b/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist deleted file mode 100644 index c75dd01..0000000 --- a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - iPhoneSimulator - - CFBundleVersion - 1 - MinimumOSVersion - 13.0 - - diff --git a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap b/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI b/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index eb34ba7..0000000 --- a/TiktokenFFI.xcframework/ios-arm64_x86_64-simulator/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7b77e759ed583479b157f56691cb37b008a5d2e92beceaccd8409414adb351b7 -size 69411896 diff --git a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h b/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h deleted file mode 100644 index 6be1674..0000000 --- a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Headers/TiktokenFFI.h +++ /dev/null @@ -1,577 +0,0 @@ -// This file was autogenerated by some hot garbage in the `uniffi` crate. -// Trust me, you don't want to mess with it! - -#pragma once - -#include -#include -#include - -// The following structs are used to implement the lowest level -// of the FFI, and thus useful to multiple uniffied crates. -// We ensure they are declared exactly once, with a header guard, UNIFFI_SHARED_H. -#ifdef UNIFFI_SHARED_H - // We also try to prevent mixing versions of shared uniffi header structs. - // If you add anything to the #else block, you must increment the version suffix in UNIFFI_SHARED_HEADER_V4 - #ifndef UNIFFI_SHARED_HEADER_V4 - #error Combining helper code from multiple versions of uniffi is not supported - #endif // ndef UNIFFI_SHARED_HEADER_V4 -#else -#define UNIFFI_SHARED_H -#define UNIFFI_SHARED_HEADER_V4 -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ - -typedef struct RustBuffer -{ - uint64_t capacity; - uint64_t len; - uint8_t *_Nullable data; -} RustBuffer; - -typedef struct ForeignBytes -{ - int32_t len; - const uint8_t *_Nullable data; -} ForeignBytes; - -// Error definitions -typedef struct RustCallStatus { - int8_t code; - RustBuffer errorBuf; -} RustCallStatus; - -// ⚠️ Attention: If you change this #else block (ending in `#endif // def UNIFFI_SHARED_H`) you *must* ⚠️ -// ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ -#endif // def UNIFFI_SHARED_H -#ifndef UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -#define UNIFFI_FFIDEF_RUST_FUTURE_CONTINUATION_CALLBACK -typedef void (*UniffiRustFutureContinuationCallback)(uint64_t, int8_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK -typedef void (*UniffiForeignFutureDroppedCallback)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_FREE -typedef void (*UniffiCallbackInterfaceFree)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -#define UNIFFI_FFIDEF_CALLBACK_INTERFACE_CLONE -typedef uint64_t (*UniffiCallbackInterfaceClone)(uint64_t - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_DROPPED_CALLBACK_STRUCT -typedef struct UniffiForeignFutureDroppedCallbackStruct { - uint64_t handle; - UniffiForeignFutureDroppedCallback _Nonnull free; -} UniffiForeignFutureDroppedCallbackStruct; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U8 -typedef struct UniffiForeignFutureResultU8 { - uint8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U8 -typedef void (*UniffiForeignFutureCompleteU8)(uint64_t, UniffiForeignFutureResultU8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I8 -typedef struct UniffiForeignFutureResultI8 { - int8_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI8; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I8 -typedef void (*UniffiForeignFutureCompleteI8)(uint64_t, UniffiForeignFutureResultI8 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U16 -typedef struct UniffiForeignFutureResultU16 { - uint16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U16 -typedef void (*UniffiForeignFutureCompleteU16)(uint64_t, UniffiForeignFutureResultU16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I16 -typedef struct UniffiForeignFutureResultI16 { - int16_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI16; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I16 -typedef void (*UniffiForeignFutureCompleteI16)(uint64_t, UniffiForeignFutureResultI16 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U32 -typedef struct UniffiForeignFutureResultU32 { - uint32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U32 -typedef void (*UniffiForeignFutureCompleteU32)(uint64_t, UniffiForeignFutureResultU32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I32 -typedef struct UniffiForeignFutureResultI32 { - int32_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I32 -typedef void (*UniffiForeignFutureCompleteI32)(uint64_t, UniffiForeignFutureResultI32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_U64 -typedef struct UniffiForeignFutureResultU64 { - uint64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultU64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_U64 -typedef void (*UniffiForeignFutureCompleteU64)(uint64_t, UniffiForeignFutureResultU64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_I64 -typedef struct UniffiForeignFutureResultI64 { - int64_t returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultI64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_I64 -typedef void (*UniffiForeignFutureCompleteI64)(uint64_t, UniffiForeignFutureResultI64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F32 -typedef struct UniffiForeignFutureResultF32 { - float returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF32; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F32 -typedef void (*UniffiForeignFutureCompleteF32)(uint64_t, UniffiForeignFutureResultF32 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_F64 -typedef struct UniffiForeignFutureResultF64 { - double returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultF64; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_F64 -typedef void (*UniffiForeignFutureCompleteF64)(uint64_t, UniffiForeignFutureResultF64 - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_RUST_BUFFER -typedef struct UniffiForeignFutureResultRustBuffer { - RustBuffer returnValue; - RustCallStatus callStatus; -} UniffiForeignFutureResultRustBuffer; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_RUST_BUFFER -typedef void (*UniffiForeignFutureCompleteRustBuffer)(uint64_t, UniffiForeignFutureResultRustBuffer - ); - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_RESULT_VOID -typedef struct UniffiForeignFutureResultVoid { - RustCallStatus callStatus; -} UniffiForeignFutureResultVoid; - -#endif -#ifndef UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FOREIGN_FUTURE_COMPLETE_VOID -typedef void (*UniffiForeignFutureCompleteVoid)(uint64_t, UniffiForeignFutureResultVoid - ); - -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_CLONE_COREBPE -uint64_t uniffi_tiktoken_fn_clone_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FREE_COREBPE -void uniffi_tiktoken_fn_free_corebpe(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_DECODE_BYTES -RustBuffer uniffi_tiktoken_fn_method_corebpe_decode_bytes(uint64_t ptr, RustBuffer tokens, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode(uint64_t ptr, RustBuffer text, RustBuffer allowed_special, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_ORDINARY -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_ordinary(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -RustBuffer uniffi_tiktoken_fn_method_corebpe_encode_with_special_tokens(uint64_t ptr, RustBuffer text, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_FN_FUNC_NEW_CORE_BPE -uint64_t uniffi_tiktoken_fn_func_new_core_bpe(RustBuffer encoder, RustBuffer special_tokens_encoder, RustBuffer pattern, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_ALLOC -RustBuffer ffi_tiktoken_rustbuffer_alloc(uint64_t size, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FROM_BYTES -RustBuffer ffi_tiktoken_rustbuffer_from_bytes(ForeignBytes bytes, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_FREE -void ffi_tiktoken_rustbuffer_free(RustBuffer buf, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUSTBUFFER_RESERVE -RustBuffer ffi_tiktoken_rustbuffer_reserve(RustBuffer buf, uint64_t additional, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U8 -void ffi_tiktoken_rust_future_poll_u8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U8 -void ffi_tiktoken_rust_future_cancel_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U8 -void ffi_tiktoken_rust_future_free_u8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U8 -uint8_t ffi_tiktoken_rust_future_complete_u8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I8 -void ffi_tiktoken_rust_future_poll_i8(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I8 -void ffi_tiktoken_rust_future_cancel_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I8 -void ffi_tiktoken_rust_future_free_i8(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I8 -int8_t ffi_tiktoken_rust_future_complete_i8(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U16 -void ffi_tiktoken_rust_future_poll_u16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U16 -void ffi_tiktoken_rust_future_cancel_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U16 -void ffi_tiktoken_rust_future_free_u16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U16 -uint16_t ffi_tiktoken_rust_future_complete_u16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I16 -void ffi_tiktoken_rust_future_poll_i16(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I16 -void ffi_tiktoken_rust_future_cancel_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I16 -void ffi_tiktoken_rust_future_free_i16(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I16 -int16_t ffi_tiktoken_rust_future_complete_i16(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U32 -void ffi_tiktoken_rust_future_poll_u32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U32 -void ffi_tiktoken_rust_future_cancel_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U32 -void ffi_tiktoken_rust_future_free_u32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U32 -uint32_t ffi_tiktoken_rust_future_complete_u32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I32 -void ffi_tiktoken_rust_future_poll_i32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I32 -void ffi_tiktoken_rust_future_cancel_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I32 -void ffi_tiktoken_rust_future_free_i32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I32 -int32_t ffi_tiktoken_rust_future_complete_i32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_U64 -void ffi_tiktoken_rust_future_poll_u64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_U64 -void ffi_tiktoken_rust_future_cancel_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_U64 -void ffi_tiktoken_rust_future_free_u64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_U64 -uint64_t ffi_tiktoken_rust_future_complete_u64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_I64 -void ffi_tiktoken_rust_future_poll_i64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_I64 -void ffi_tiktoken_rust_future_cancel_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_I64 -void ffi_tiktoken_rust_future_free_i64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_I64 -int64_t ffi_tiktoken_rust_future_complete_i64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F32 -void ffi_tiktoken_rust_future_poll_f32(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F32 -void ffi_tiktoken_rust_future_cancel_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F32 -void ffi_tiktoken_rust_future_free_f32(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F32 -float ffi_tiktoken_rust_future_complete_f32(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_F64 -void ffi_tiktoken_rust_future_poll_f64(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_F64 -void ffi_tiktoken_rust_future_cancel_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_F64 -void ffi_tiktoken_rust_future_free_f64(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_F64 -double ffi_tiktoken_rust_future_complete_f64(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_RUST_BUFFER -void ffi_tiktoken_rust_future_poll_rust_buffer(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_RUST_BUFFER -void ffi_tiktoken_rust_future_cancel_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_RUST_BUFFER -void ffi_tiktoken_rust_future_free_rust_buffer(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_RUST_BUFFER -RustBuffer ffi_tiktoken_rust_future_complete_rust_buffer(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_POLL_VOID -void ffi_tiktoken_rust_future_poll_void(uint64_t handle, UniffiRustFutureContinuationCallback _Nonnull callback, uint64_t callback_data -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_CANCEL_VOID -void ffi_tiktoken_rust_future_cancel_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_FREE_VOID -void ffi_tiktoken_rust_future_free_void(uint64_t handle -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_RUST_FUTURE_COMPLETE_VOID -void ffi_tiktoken_rust_future_complete_void(uint64_t handle, RustCallStatus *_Nonnull out_status -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_FUNC_NEW_CORE_BPE -uint16_t uniffi_tiktoken_checksum_func_new_core_bpe(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_DECODE_BYTES -uint16_t uniffi_tiktoken_checksum_method_corebpe_decode_bytes(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_ORDINARY -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_ordinary(void - -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -#define UNIFFI_FFIDEF_UNIFFI_TIKTOKEN_CHECKSUM_METHOD_COREBPE_ENCODE_WITH_SPECIAL_TOKENS -uint16_t uniffi_tiktoken_checksum_method_corebpe_encode_with_special_tokens(void - -); -#endif -#ifndef UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -#define UNIFFI_FFIDEF_FFI_TIKTOKEN_UNIFFI_CONTRACT_VERSION -uint32_t ffi_tiktoken_uniffi_contract_version(void - -); -#endif - diff --git a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist b/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist deleted file mode 100644 index 56e4e40..0000000 --- a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Info.plist +++ /dev/null @@ -1,28 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - TiktokenFFI - CFBundleIdentifier - com.tiktoken.TiktokenFFI - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - TiktokenFFI - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0.0 - CFBundleSupportedPlatforms - - MacOSX - - CFBundleVersion - 1 - MinimumOSVersion - 10.15 - - diff --git a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap b/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap deleted file mode 100644 index 4a57a94..0000000 --- a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/Modules/module.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -framework module TiktokenFFI { - header "TiktokenFFI.h" - export * -} diff --git a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI b/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI deleted file mode 100644 index 8977a48..0000000 --- a/TiktokenFFI.xcframework/macos-arm64_x86_64/TiktokenFFI.framework/TiktokenFFI +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:57c82030685a591b7d69058ce8b9f6cd90824332b60f6f4d133be0d3d3c5c43d -size 70092376