diff --git a/documentation/contributing/rust.md b/documentation/contributing/rust.md index c47548310b..6b1283773c 100644 --- a/documentation/contributing/rust.md +++ b/documentation/contributing/rust.md @@ -62,6 +62,8 @@ src/extension/flow-php-ext/ │ ├── format.rs # Floe binary format primitives │ ├── hydrate.rs # Row hydration against a schema │ ├── cast.rs # Value casting +│ ├── json_check.rs # JSON validation shared by casting and CSV inference +│ ├── csv/ # CSV tokenizer, reader and schema-inference fold │ ├── plan.rs # Per-column plan resolved once per schema │ ├── ctx.rs # Shared module context │ ├── values.rs # Zval <-> PHP value helpers diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 75bc0a4d67..dca41dedcf 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -221,6 +221,23 @@ after it shift by one. `Stage::physical` is new - see the core documentation. | `new Explain\TreeLayout($details, declarations: true)` | `new Explain\Outline(declarations: true)`, the layout takes no arguments | | `new Explain\BoxLayout($details)` | `new Explain\BoxLayout()` | +### 22) `flow-php/etl-adapter-csv` - an escaped or bare enclosure no longer merges a record with the next line + +| Input (3 lines) | Before | After | +|------------------------------|----------------------------------------|---------------------------------------------------| +| `a,b` / `"x\"y",1` / `"p",2` | 1 row: `{"a":"x\\\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\\\"y","b":1}`, `{"a":"p","b":2}` | +| `a,b` / `x"y,1` / `"p",2` | 1 row: `{"a":"x\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\"y","b":1}`, `{"a":"p","b":2}` | + +### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVLineReader` takes the separator and escape + +| Before | After | +|----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `new CSVLineReader($enclosure, $charactersReadInLine, $removeBOM)` | `new CSVLineReader($enclosure, $separator, $escape, $charactersReadInLine, $removeBOM)` | +| `new CSVOpenSource($stream, $dialect, $encoder, $lineReader)` | `new PhpCSVOpenSource($stream, $encoder, $lineReader)`; `CSVOpenSource` is its interface | +| `$open->stream`, `$open->dialect`, `$open->encoder`, `$open->lineReader` | removed | +| `CSVFileReader::samples()` yields `Generator`s | yields `CSVFileSample` (`IteratorAggregate`); `$unit->getIterator()` for the generator | +| `CSVFileReader::sample($source)` | `new CSVFileSample($opener, $source)` | + --- ## Upgrading from 0.43.x to 0.44.x diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php new file mode 100644 index 0000000000..6b3ae0fb27 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php @@ -0,0 +1,81 @@ +blanks = str_replace($separator, '', self::BLANKS); + $this->specials = $escape === $enclosure ? $enclosure : $enclosure . $escape; + } + + public function endsOutsideAnEnclosure(string $buffer): bool + { + $length = strlen($buffer); + $position = 0; + + while (true) { + $fieldStart = $position + strspn($buffer, $this->blanks, $position); + + if ($fieldStart < $length && $buffer[$fieldStart] === $this->enclosure) { + $position = $fieldStart + 1; + + while (true) { + $position += strcspn($buffer, $this->specials, $position); + + if ($position >= $length) { + return false; + } + + if ($buffer[$position] !== $this->enclosure) { + $position += 2; + + continue; + } + + if (($position + 1) < $length && $buffer[$position + 1] === $this->enclosure) { + $position += 2; + + continue; + } + + $position++; + + break; + } + } + + $nextSeparator = strpos($buffer, $this->separator, $position); + + if ($nextSeparator === false) { + return true; + } + + $position = $nextSeparator + 1; + } + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php index 1bbfb691d2..8dc8f6d79b 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php @@ -89,10 +89,8 @@ public function decode(array $batch): array $maps = []; foreach ($batch as $line) { - $fields = array_values(array_map( - static fn(mixed $field): ?string => is_string($field) ? $field : null, - str_getcsv($line, $this->separator, $this->enclosure, $this->escape), - )); + /** @var list $fields */ + $fields = str_getcsv($line, $this->separator, $this->enclosure, $this->escape); if ($this->headers === null) { if ($this->withHeader) { diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php index e937af1719..2d90dc60a4 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php @@ -79,28 +79,14 @@ public function header(): CSVHeader } /** - * @return Generator - */ - public function sample(SourceFile $source): Generator - { - $open = $this->opener->open($source); - - try { - yield from $open->records(); - } finally { - $open->close(); - } - } - - /** - * $rowBudget is deliberately unused: sample() is lazy and SchemaInferrer stops advancing it. + * $rowBudget is unused: SchemaInferrer hands each unit its remaining budget through sniffColumnTypes(). * - * @return Generator> + * @return Generator */ public function samples(int $rowBudget): iterable { foreach ($this->sources as $source) { - yield $this->sample($source); + yield new CSVFileSample($this->opener, $source); } } } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php new file mode 100644 index 0000000000..10bb522b1c --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php @@ -0,0 +1,56 @@ + + */ +final readonly class CSVFileSample implements IteratorAggregate, SniffsColumnTypes +{ + public function __construct( + private CSVSourceOpener $opener, + private SourceFile $source, + ) {} + + /** + * The source is opened on the first advance; abandoning the generator closes it. + * + * @return Generator + */ + public function getIterator(): Generator + { + $open = $this->opener->open($this->source); + + try { + yield from $open->records(); + } finally { + $open->close(); + } + } + + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes { + $open = $this->opener->open($this->source); + + try { + return $open->sniff($names, $rowBudget, $inference, $typer); + } finally { + $open->close(); + } + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php index 88d6011fa0..482134978c 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php @@ -11,18 +11,23 @@ use function str_contains; use function str_starts_with; use function substr; -use function substr_count; final readonly class CSVLineReader { + private CSVRecordBoundary $boundary; + /** * @param null|int<1, max> $charactersReadInLine */ public function __construct( private string $enclosure, + string $separator = ',', + string $escape = '\\', private ?int $charactersReadInLine = null, private bool $removeBOM = true, - ) {} + ) { + $this->boundary = new CSVRecordBoundary($enclosure, $separator, $escape); + } /** * @return \Generator @@ -42,7 +47,7 @@ public function readLines(SourceStream $stream): Generator $lineNumber++; $buffer = ''; } else { - if ($this->isCompleteCSVRecord($buffer)) { + if ($this->boundary->isComplete($buffer)) { yield $this->removeBOM && $lineNumber === 0 ? $this->removeBOMFromLine(rtrim($buffer, "\r\n")) : rtrim($buffer, "\r\n"); @@ -61,19 +66,6 @@ public function readLines(SourceStream $stream): Generator } } - /** - * Check if the current buffer contains a complete CSV record - * by counting enclosures and ensuring they are properly paired. - */ - private function isCompleteCSVRecord(string $buffer): bool - { - if (!str_contains($buffer, $this->enclosure)) { - return true; - } - - return (substr_count($buffer, $this->enclosure) % 2) === 0; - } - /** * Remove Byte Order Mark (BOM) from the beginning of a line if present. */ diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php index cad701dc86..e030ac48df 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php @@ -5,51 +5,34 @@ namespace Flow\ETL\Adapter\CSV; use Flow\ETL\Row\RawRowValues; -use Flow\Filesystem\SourceStream; +use Flow\ETL\Schema\Inference\ColumnTypes; +use Flow\ETL\Schema\Inference\SchemaInference; +use Flow\Types\Type\TypeNarrower; use Generator; -final readonly class CSVOpenSource +interface CSVOpenSource { - public function __construct( - public SourceStream $stream, - public CSVDialect $dialect, - public CSVEncoder $encoder, - public CSVLineReader $lineReader, - ) {} - - public function close(): void - { - $this->stream->close(); - } + public function close(): void; /** * This instance is consumed afterwards. * * @return list */ - public function columns(): array - { - foreach ($this->lineReader->readLines($this->stream) as $line) { - $this->encoder->decode([$line]); - - break; - } - - return $this->encoder->headers() ?? []; - } + public function columns(): array; /** - * CSVLineReader::readLines() already joins a quoted multi-line record, so never re-split or re-join here. * This instance is consumed afterwards. * * @return Generator */ - public function records(): Generator - { - foreach ($this->lineReader->readLines($this->stream) as $line) { - foreach ($this->encoder->decode([$line]) as $values) { - yield $values; - } - } - } + public function records(): Generator; + + /** + * SchemaInferrer::sniff() over records(). This instance is consumed afterwards. + * + * @param list $names + * @param int<0, max>|-1 $rowBudget + */ + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes; } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php new file mode 100644 index 0000000000..b89f0de05f --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php @@ -0,0 +1,56 @@ +pattern = + '/\A(?:' . $blanks . '(?:' . $enclosedField . '|' . $unenclosedField . ')' . $fieldEnd . ')*+\z/s'; + $this->scan = new CSVEnclosureScan($separator, $enclosure, $escape); + } + + public function isComplete(string $buffer): bool + { + if (!str_contains($buffer, $this->enclosure)) { + return true; + } + + $matched = preg_match($this->pattern, $buffer); + + // PCRE gives up at about 142k fields in one record; reading that as "incomplete" would glue every following line + return $matched === false ? $this->scan->endsOutsideAnEnclosure($buffer) : $matched === 1; + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php index 2bc90e9791..5e27c6390b 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php @@ -28,9 +28,23 @@ public function open(SourceFile $source): CSVOpenSource $this->options->escape ?? $detected->escape, ); - return new CSVOpenSource( + if (NativeCSVOpenSource::isSupported()) { + return new NativeCSVOpenSource( + $stream, + new RustCSVReaderNative( + $dialect->separator, + $dialect->enclosure, + $dialect->escape, + $this->options->withHeader, + $this->options->emptyToNull, + $this->options->removeBOM, + ), + $this->options->charactersReadInLine, + ); + } + + return new PhpCSVOpenSource( $stream, - $dialect, new CSVEncoder( withHeader: $this->options->withHeader, separator: $dialect->separator, @@ -38,7 +52,13 @@ public function open(SourceFile $source): CSVOpenSource escape: $dialect->escape, emptyToNull: $this->options->emptyToNull, ), - new CSVLineReader($dialect->enclosure, $this->options->charactersReadInLine, $this->options->removeBOM), + new CSVLineReader( + $dialect->enclosure, + $dialect->separator, + $dialect->escape, + $this->options->charactersReadInLine, + $this->options->removeBOM, + ), ); } catch (Throwable $e) { $stream->close(); diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php new file mode 100644 index 0000000000..ab3da4a2c1 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php @@ -0,0 +1,158 @@ + + */ + private int $chunkSize; + + /** + * @param null|int<1, max> $charactersReadInLine + */ + public function __construct( + private SourceStream $stream, + private RustCSVReaderNative $reader, + ?int $charactersReadInLine = null, + ) { + // a local file ignores the read length, as NativeLocalSourceStream::readLines() already does on the PHP path; + // for S3/Azure it is the range-request size - and from_csv() defaults it to 10 MB, which would be the chunk + $this->chunkSize = $stream instanceof NativeLocalSourceStream + ? self::CHUNK + : $charactersReadInLine ?? self::CHUNK; + } + + public static function isSupported(): bool + { + return extension_loaded('flow_php') && class_exists(RustCSVReaderNative::class, false); + } + + public function close(): void + { + $this->stream->close(); + } + + public function columns(): array + { + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + + $headers = $this->reader->headers(); + + if ($headers !== []) { + return $headers; + } + } + + $this->reader->finish(); + + return $this->reader->headers(); + } + + public function records(): Generator + { + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + + while (($batch = $this->reader->next(self::BATCH)) !== []) { + foreach ($batch as $values) { + yield $values; + } + } + } + + $this->reader->finish(); + + while (($batch = $this->reader->next(self::BATCH)) !== []) { + foreach ($batch as $values) { + yield $values; + } + } + } + + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes + { + // the Rust fold is StringTypeNarrower without its HTML and XML rungs - a DOMDocument per cell is not worth porting + if (!$typer instanceof StringTypeNarrower || $typer->emitsType(type_html()) || $typer->emitsType(type_xml())) { + return (new SchemaInferrer($inference, $typer))->sniff($names, $this->records(), $rowBudget); + } + + $fold = new RustColumnFoldNative($names, array_map( + static fn(Type $type): string => $type->toString(), + array_values(array_filter( + [ + type_json(), + type_uuid(), + type_float(), + type_integer(), + type_datetime(), + type_date(), + type_boolean(), + type_time_zone(), + ], + $typer->emitsType(...), + )), + )); + + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + $this->reader->fold($fold, $rowBudget === -1 ? -1 : max(0, $rowBudget - $fold->rows())); + + if ($rowBudget !== -1 && $fold->rows() >= $rowBudget) { + break; + } + } + + if ($rowBudget === -1 || $fold->rows() < $rowBudget) { + $this->reader->finish(); + $this->reader->fold($fold, $rowBudget === -1 ? -1 : max(0, $rowBudget - $fold->rows())); + } + + return ColumnTypes::fromColumnTypes( + array_map(TypeFactory::fromString(...), $fold->types()), + $fold->rows(), + $typer, + ); + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php new file mode 100644 index 0000000000..5af14d0595 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php @@ -0,0 +1,63 @@ +stream->close(); + } + + /** + * This instance is consumed afterwards. + * + * @return list + */ + public function columns(): array + { + foreach ($this->lineReader->readLines($this->stream) as $line) { + $this->encoder->decode([$line]); + + break; + } + + return $this->encoder->headers() ?? []; + } + + /** + * CSVLineReader::readLines() already joins a quoted multi-line record, so never re-split or re-join here. + * This instance is consumed afterwards. + * + * @return Generator + */ + public function records(): Generator + { + foreach ($this->lineReader->readLines($this->stream) as $line) { + foreach ($this->encoder->decode([$line]) as $values) { + yield $values; + } + } + } + + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes + { + return (new SchemaInferrer($inference, $typer))->sniff($names, $this->records(), $rowBudget); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php index 1c9f025979..c01f3cda5b 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php @@ -4,18 +4,43 @@ namespace Flow\ETL\Adapter\CSV\Tests\Context; +use Flow\ETL\Adapter\CSV\CSVDialect; +use Flow\ETL\Adapter\CSV\CSVEncoder; use Flow\ETL\Adapter\CSV\CSVFileReader; +use Flow\ETL\Adapter\CSV\CSVFileSample; +use Flow\ETL\Adapter\CSV\CSVLineReader; use Flow\ETL\Adapter\CSV\CSVOpenSource; use Flow\ETL\Adapter\CSV\CSVReadOptions; use Flow\ETL\Adapter\CSV\CSVSourceOpener; +use Flow\ETL\Adapter\CSV\NativeCSVOpenSource; +use Flow\ETL\Adapter\CSV\PhpCSVOpenSource; +use Flow\ETL\Adapter\CSV\RustCSVReaderNative; use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Row\RawRowValues; +use Flow\ETL\Schema; +use Flow\ETL\Schema\Inference\ColumnTypes; +use Flow\ETL\Schema\Inference\SchemaInference; +use Flow\ETL\Schema\Inference\SchemaInferrer; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; +use Flow\Filesystem\Local\MemoryFilesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path\Filter\OnlyFiles; +use Flow\Filesystem\SourceStream; +use Flow\Types\Type\Native\String\StringTypeNarrower; +use Flow\Types\Type\TypeNarrower; +use Generator; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; +use SplFileInfo; +use function Flow\ETL\Adapter\CSV\csv_detect_separator; +use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; +use function str_ends_with; +use function strlen; +use function substr; /** * Resolves the adapter's test fixtures and builds readers over them, so no test needs a private helper. @@ -47,6 +72,192 @@ public static function open( return (new CSVSourceOpener($filesystem, $options))->open(self::source($fixture)); } + /** + * Every *.csv under Fixtures/, keyed and valued by its path relative to Fixtures/. + * + * @return Generator + */ + public static function fixtures(): Generator + { + $root = self::path(''); + + /** @var SplFileInfo $file */ + foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator( + $root, + RecursiveDirectoryIterator::SKIP_DOTS, + )) as $file) { + if (str_ends_with($file->getFilename(), '.csv')) { + $fixture = substr($file->getPathname(), strlen($root)); + + yield $fixture => [$fixture]; + } + } + } + + /** + * The dialect CSVSourceOpener would resolve: pinned options first, detection for the rest. + */ + public static function dialect(SourceStream $stream, CSVReadOptions $options = new CSVReadOptions()): CSVDialect + { + $detected = csv_detect_separator($stream); + + return new CSVDialect( + $options->separator ?? $detected->separator, + $options->enclosure ?? $detected->enclosure, + $options->escape ?? $detected->escape, + ); + } + + /** + * Requires the extension. + */ + public static function openNative( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): NativeCSVOpenSource { + return self::openNativeStream($filesystem->readFrom(path_real(self::path($fixture))), $options); + } + + /** + * The native open source over an already-open stream - requires the extension. + */ + public static function openNativeStream( + SourceStream $stream, + CSVReadOptions $options = new CSVReadOptions(), + ): NativeCSVOpenSource { + $dialect = self::dialect($stream, $options); + + return new NativeCSVOpenSource( + $stream, + new RustCSVReaderNative( + $dialect->separator, + $dialect->enclosure, + $dialect->escape, + $options->withHeader, + $options->emptyToNull, + $options->removeBOM, + ), + $options->charactersReadInLine, + ); + } + + public static function openPhp( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): PhpCSVOpenSource { + $stream = $filesystem->readFrom(path_real(self::path($fixture))); + $dialect = self::dialect($stream, $options); + + return new PhpCSVOpenSource( + $stream, + new CSVEncoder( + withHeader: $options->withHeader, + separator: $dialect->separator, + enclosure: $dialect->enclosure, + escape: $dialect->escape, + emptyToNull: $options->emptyToNull, + ), + new CSVLineReader( + $dialect->enclosure, + $dialect->separator, + $dialect->escape, + $options->charactersReadInLine, + $options->removeBOM, + ), + ); + } + + /** + * Every record's values and metadata, for strict comparison across paths - consumes the source. + * + * @return list, array}> + */ + public static function records(CSVOpenSource $open): array + { + $records = []; + + foreach ($open->records() as $record) { + $records[] = [$record->values, $record->metadata]; + } + + return $records; + } + + /** + * The schema CSVExtractor infers - through CSVFileReader::samples(), native when the extension is loaded. + */ + public static function infer(SchemaInference $inference, string ...$fixtures): Schema + { + $reader = self::reader(sources: self::sources(...$fixtures)); + + return (new SchemaInferrer($inference, new StringTypeNarrower($inference->candidates()->toArray())))->infer( + $reader->header()->names, + $reader->samples($inference->sampleSize), + ); + } + + /** + * The canonical PHP fold: the PHP open source's records observed row by row, whatever the opener would pick. + */ + public static function inferPhp(SchemaInference $inference, string ...$fixtures): Schema + { + $names = []; + + foreach ($fixtures as $fixture) { + $open = self::openPhp($fixture); + $names = $open->columns(); + $open->close(); + + if ($names !== []) { + break; + } + } + + $samples = []; + + foreach ($fixtures as $fixture) { + $samples[] = self::phpRecords($fixture); + } + + return (new SchemaInferrer($inference, new StringTypeNarrower($inference->candidates()->toArray())))->infer( + $names, + $samples, + ); + } + + /** + * The PHP open source's records, opened on the first advance and closed when abandoned - a SchemaSampler unit. + * + * @return Generator + */ + public static function phpRecords(string $fixture): Generator + { + $open = self::openPhp($fixture); + + try { + yield from $open->records(); + } finally { + $open->close(); + } + } + + public static function memory(string $content): MemoryFilesystem + { + $filesystem = memory_filesystem(); + $stream = $filesystem->writeTo(self::memorySource()->path); + $stream->append($content); + $stream->close(); + + return $filesystem; + } + + public static function memorySource(): SourceFile + { + return new SourceFile(path('memory://source.csv')); + } + public static function path(string $fixture): string { return __DIR__ . '/../Fixtures/' . $fixture; @@ -66,6 +277,38 @@ public static function reader( return new CSVFileReader(new CSVSourceOpener($filesystem, $options), $sources); } + public static function sample( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): CSVFileSample { + return new CSVFileSample(new CSVSourceOpener($filesystem, $options), self::source($fixture)); + } + + /** + * The fixture's partial ColumnTypes observed row by row over the PHP records, and sniffed by CSVFileSample - + * natively when the extension is loaded. + * + * @param int<0, max>|-1 $rowBudget + * + * @return array{ColumnTypes, ColumnTypes} + */ + public static function sniffBothWays( + string $fixture, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): array { + $open = self::openPhp($fixture); + $names = $open->columns(); + $open->close(); + + return [ + (new SchemaInferrer($inference, $typer))->sniff($names, self::phpRecords($fixture), $rowBudget), + self::sample($fixture)->sniffColumnTypes($names, $rowBudget, $inference, $typer), + ]; + } + public static function source(string $fixture): SourceFile { return new SourceFile(path_real(self::path($fixture))); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php new file mode 100644 index 0000000000..c1483b5198 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php @@ -0,0 +1,38 @@ + + */ + public static function buffers(): Generator + { + yield 'no enclosure' => [',', '"', '\\', 'a,b', true]; + yield 'closed enclosure' => [',', '"', '\\', '"a",b', true]; + yield 'open enclosure' => [',', '"', '\\', '"a', false]; + yield 'open enclosure across a line' => [',', '"', '\\', "\"a\nb", false]; + yield 'closed across a line' => [',', '"', '\\', "\"a\nb\",c", true]; + yield 'escaped enclosure keeps it open' => [',', '"', '\\', '"x\"y', false]; + yield 'escape as the last byte' => [',', '"', '\\', '"x\\', false]; + yield 'doubled enclosure keeps it open' => [',', '"', '\\', '"a""', false]; + yield 'doubled then closed' => [',', '"', '\\', '"a"""', true]; + yield 'enclosure inside an unenclosed field' => [',', '"', '\\', 'x"y,1', true]; + yield 'blanks before an opening enclosure' => [',', '"', '\\', "a, \t\"b", false]; + yield 'carriage return before an opening enclosure' => [',', '"', '\\', "a,\r\"b", false]; + yield 'junk after a closing enclosure' => [',', '"', '\\', '"a"x"y', true]; + yield 'empty escape' => [',', '"', '', '"x\",1', true]; + yield 'escape equal to the enclosure' => [',', '"', '"', '"a""', false]; + yield 'custom separator and enclosure' => [';', "'", '\\', "x,'y;1", true]; + yield 'custom enclosure left open' => [';', "'", '\\', "a;'b", false]; + yield 'whitespace separator is not a blank' => ["\t", '"', '\\', "a\t\t\"b", false]; + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php new file mode 100644 index 0000000000..694a6836b1 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php @@ -0,0 +1,84 @@ +wrapped->appendTo($path); + } + + public function getSystemTmpDir(): Path + { + return $this->wrapped->getSystemTmpDir(); + } + + public function list(Path $path, Filter $pathFilter = new KeepAll()): Generator + { + yield from $this->wrapped->list($path, $pathFilter); + } + + public function mount(): Mount + { + return $this->wrapped->mount(); + } + + public function mv(Path $from, Path $to): bool + { + return $this->wrapped->mv($from, $to); + } + + public function readFrom(Path $path): SourceStream + { + $stream = $this->wrapped->readFrom($path); + + return $this->lastStream = new LengthCapturingSourceStream( + $stream->read(max(1, (int) $stream->size()), 0), + $path, + ); + } + + public function rm(Path $path): bool + { + return $this->wrapped->rm($path); + } + + public function status(Path $path): ?FileStatus + { + return $this->wrapped->status($path); + } + + public function supports(Path $path): bool + { + return $this->wrapped->supports($path); + } + + public function writeTo(Path $path): DestinationStream + { + return $this->wrapped->writeTo($path); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php index 008c96f52b..a7298f5536 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php @@ -19,6 +19,11 @@ final class LengthCapturingSourceStream implements SourceStream */ public array $capturedLengths = []; + /** + * @var list the length argument captured on each iterate() call + */ + public array $capturedIterateLengths = []; + public function __construct( private readonly string $contents, private readonly Path $path, @@ -38,6 +43,8 @@ public function isOpen(): bool public function iterate(int $length = 1): Generator { + $this->capturedIterateLengths[] = $length; + for ($i = 0; $i < strlen($this->contents); $i += $length) { yield substr($this->contents, $i, $length); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv new file mode 100644 index 0000000000..047e70a5fc --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv @@ -0,0 +1,3 @@ +a,b +x"y,1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv new file mode 100644 index 0000000000..398a0836db --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv @@ -0,0 +1,2 @@ +"a""b",1 +"c",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv new file mode 100644 index 0000000000..004cbefa36 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv @@ -0,0 +1,3 @@ +a,b +"x\"y",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv new file mode 100644 index 0000000000..004cbefa36 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv @@ -0,0 +1,3 @@ +a,b +"x\"y",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv new file mode 100644 index 0000000000..5529c19a29 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv @@ -0,0 +1,4 @@ +a,b +"x\"y +z",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv new file mode 100644 index 0000000000..e83306b1ce --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv @@ -0,0 +1,3 @@ +a,b +"x\"y,z",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv new file mode 100644 index 0000000000..8d72765c7c --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv @@ -0,0 +1,4 @@ +id,pad,n,dup,dup +1, 5 ,NULL,1,x +2,7,nil,2,y +3,8,5,3,z diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php index af2d2932f4..b9a4404a74 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php @@ -22,6 +22,7 @@ use Flow\Filesystem\Tests\OperatingSystem; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\DataProviderExternal; use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; @@ -30,6 +31,8 @@ use function array_map; use function array_sum; use function count; +use function fclose; +use function fgetcsv; use function Flow\ETL\Adapter\CSV\from_csv; use function Flow\ETL\Adapter\CSV\to_csv; use function Flow\ETL\DSL\config; @@ -46,9 +49,14 @@ use function Flow\ETL\DSL\schema_metadata; use function Flow\ETL\DSL\schema_to_ascii; use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\native_local_filesystem; use function Flow\Filesystem\DSL\path_real; +use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_html; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_xml; +use function fopen; use function iterator_to_array; use function max; use function sort; @@ -1126,4 +1134,139 @@ private function ensureBOMExists(string $path, string $BOM): bool return $contents === $BOM; } + + public function test_an_escaped_quote_does_not_glue_the_following_record(): void + { + static::assertSame( + [['a' => 'x\"y', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_then_row.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_a_bare_quote_in_an_unenclosed_field_does_not_glue_the_following_record(): void + { + static::assertSame( + [['a' => 'x"y', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('bare_quote_then_row.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_next_to_an_embedded_newline_keeps_one_record(): void + { + static::assertSame( + [['a' => "x\\\"y\nz", 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_with_newline.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_next_to_an_embedded_separator_keeps_one_record(): void + { + static::assertSame( + [['a' => 'x\"y,z', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_with_separator.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_read_without_an_escape_character_closes_the_field(): void + { + static::assertSame( + [['a' => 'x\y"', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_no_escape_char.csv'))->withEscape('')) + ->fetch() + ->toArray(), + ); + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_record_counts_match_fgetcsv_for_every_fixture(string $fixture): void + { + $path = CSVFixtureContext::path($fixture); + $stream = native_local_filesystem()->readFrom(path_real($path)); + $dialect = CSVFixtureContext::dialect($stream); + $stream->close(); + + $handle = fopen($path, 'rb'); + $records = 0; + + while (fgetcsv($handle, 0, $dialect->separator, $dialect->enclosure, $dialect->escape) !== false) { + $records++; + } + + fclose($handle); + + static::assertSame( + max(0, $records - 1), + df() + ->read( + from_csv($path) + ->withSeparator($dialect->separator) + ->withEnclosure($dialect->enclosure) + ->withEscape($dialect->escape), + ) + ->fetch() + ->count(), + ); + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_native_and_php_inference_agree_on_every_fixture(string $fixture): void + { + static::assertEquals( + CSVFixtureContext::inferPhp(infer_schema()->build(), $fixture), + CSVFixtureContext::infer(infer_schema()->build(), $fixture), + ); + } + + public function test_sample_size_minus_one_agrees_on_both_paths(): void + { + $inference = infer_schema()->sampleSize(-1)->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } + + public function test_union_by_name_over_a_glob_agrees_on_both_paths(): void + { + $inference = infer_schema()->unionByName()->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'columns_diverge/a.csv', 'columns_diverge/b.csv'), + CSVFixtureContext::infer($inference, 'columns_diverge/a.csv', 'columns_diverge/b.csv'), + ); + } + + public function test_a_restricted_candidate_set_agrees_on_both_paths(): void + { + $inference = infer_schema()->types(type_integer(), type_boolean())->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } + + public function test_html_or_xml_candidates_fall_back_to_the_php_path(): void + { + $inference = infer_schema()->types(type_html(), type_xml(), type_integer())->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php index 565eefe3cb..6041f355ce 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php @@ -10,7 +10,6 @@ use Flow\Filesystem\Local\NativeLocalFilesystem; use PHPUnit\Framework\Attributes\TestWith; -use function array_keys; use function count; use function iterator_to_array; @@ -110,31 +109,6 @@ public function test_header_stops_at_the_first_source_with_a_header(): void static::assertStringEndsWith('a.csv', $header->source); } - public function test_sample_closes_its_stream_when_abandoned(): void - { - $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $sample = CSVFixtureContext::reader($counting)->sample(CSVFixtureContext::source('five_rows.csv')); - - $sample->current(); - unset($sample); - - static::assertSame(1, $counting->closedStreams()); - } - - public function test_sample_yields_every_row_of_a_source(): void - { - $records = iterator_to_array( - CSVFixtureContext::reader()->sample(CSVFixtureContext::source('ragged.csv')), - false, - ); - - static::assertCount(3, $records); - - foreach ($records as $record) { - static::assertSame(['id', 'name', 'v'], array_keys($record->values)); - } - } - public function test_samples_does_not_ration_the_row_budget(): void { $units = iterator_to_array( @@ -160,7 +134,7 @@ public function test_samples_walks_the_sources_in_listing_order(): void static::assertSame([['id' => '1', 'name' => 'a'], ['id' => '3', 'label' => 'c']], $first); } - public function test_samples_yields_one_unstarted_generator_per_source(): void + public function test_samples_yields_one_unstarted_unit_per_source(): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); $units = iterator_to_array( @@ -170,12 +144,12 @@ public function test_samples_yields_one_unstarted_generator_per_source(): void $beforeAdvancing = $counting->readFromCalls; - $units[0]->current(); + $units[0]->getIterator()->current(); $afterAdvancingOne = $counting->readFromCalls; static::assertCount(2, $units); - static::assertSame(0, $beforeAdvancing, 'samples() yields UNSTARTED generators'); + static::assertSame(0, $beforeAdvancing, 'samples() yields UNSTARTED units'); static::assertSame(1, $afterAdvancingOne, 'advancing the first unit opens exactly one source'); } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php new file mode 100644 index 0000000000..c20e5b80ba --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php @@ -0,0 +1,93 @@ +values)); + } + } + + public function test_an_abandoned_iteration_closes_its_stream(): void + { + $counting = new CountingFilesystem(new NativeLocalFilesystem()); + $iterator = CSVFixtureContext::sample('five_rows.csv', $counting)->getIterator(); + + $iterator->current(); + unset($iterator); + + static::assertSame(1, $counting->closedStreams()); + } + + /** + * @param int<0, max>|-1 $rowBudget + */ + #[TestWith([-1, false])] + #[TestWith([2, false])] + #[TestWith([100, false])] + #[TestWith([-1, true])] + public function test_sniffing_equals_observing_its_records(int $rowBudget, bool $htmlCandidate): void + { + $inference = $htmlCandidate + ? infer_schema()->types(type_html(), type_integer())->build() + : infer_schema()->build(); + + [$observed, $sniffed] = CSVFixtureContext::sniffBothWays( + 'orders_flow.csv', + $rowBudget, + $inference, + new StringTypeNarrower($inference->candidates()->toArray()), + ); + + static::assertSame($observed->rows(), $sniffed->rows()); + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } + + public function test_sniffing_narrows_with_the_narrower_it_is_handed(): void + { + $inference = infer_schema()->build(); + [$observed, $sniffed] = CSVFixtureContext::sniffBothWays( + 'orders_flow.csv', + -1, + $inference, + new StringTypeNarrower([type_string()]), + ); + + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php index eefee60676..8412bd6536 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php @@ -20,7 +20,7 @@ public function test_reading_csv_with_custom_character_limit(): void $path = __DIR__ . '/../Fixtures/more_than_1000_characters_per_line.csv'; $stream = NativeLocalSourceStream::open(path_real($path)); - $reader = new CSVLineReader('"', 2000); + $reader = new CSVLineReader('"', charactersReadInLine: 2000); $lines = iterator_to_array($reader->readLines($stream)); static::assertCount(2, $lines); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php index 483ac1c6b3..ca2cce5379 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php @@ -23,7 +23,6 @@ public function test_a_pinned_separator_wins_over_detection(): void ))->open(CSVFixtureContext::source('semicolon.csv')); try { - static::assertSame(',', $open->dialect->separator); static::assertSame(['id;name'], $open->columns()); } finally { $open->close(); @@ -69,7 +68,7 @@ public function test_open_detects_the_dialect(): void ))->open(CSVFixtureContext::source('semicolon.csv')); try { - static::assertSame(';', $open->dialect->separator); + static::assertSame(['id', 'name'], $open->columns()); } finally { $open->close(); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php new file mode 100644 index 0000000000..34bd1d36b6 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php @@ -0,0 +1,335 @@ +close(); + + static::assertSame(1, $counting->closedStreams()); + } + + public function test_columns_of_an_empty_source_is_empty(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('empty.csv'); + + try { + static::assertSame([], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_reads_a_quoted_multiline_header_as_one_record(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('multiline_header.csv'); + + try { + static::assertSame(['id', "na\nme"], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_reports_the_resolved_header(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('header_only.csv'); + + try { + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_without_a_header_line_are_generated(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('two_columns.csv', options: new CSVReadOptions(withHeader: false)); + + try { + static::assertSame(['e00', 'e01'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_records_yields_every_row_with_the_full_key_set(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('ragged.csv'); + + try { + $records = iterator_to_array($open->records(), false); + + static::assertCount(3, $records); + + foreach ($records as $record) { + static::assertSame(['id', 'name', 'v'], array_keys($record->values)); + } + } finally { + $open->close(); + } + } + + public function test_records_yields_one_logical_record_per_iteration(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('multiline_strings.csv'); + + try { + $records = iterator_to_array($open->records(), false); + + static::assertNotSame([], $records); + static::assertStringContainsString("\n", implode('', array_map('strval', $records[0]->values))); + } finally { + $open->close(); + } + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_native_and_php_open_sources_yield_identical_records(string $fixture): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp($fixture); + $native = CSVFixtureContext::openNative($fixture); + + try { + static::assertSame(CSVFixtureContext::records($php), CSVFixtureContext::records($native)); + } finally { + $php->close(); + $native->close(); + } + } + + /** + * @param positive-int $chunkSize + */ + #[TestWith([1])] + #[TestWith([7])] + #[TestWith([4096])] + public function test_records_are_identical_across_chunk_sizes(int $chunkSize): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $contents = (string) file_get_contents(CSVFixtureContext::path('multiline_strings.csv')); + + static::assertSame( + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )), + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + (new CSVReadOptions())->withCharactersReadInLine($chunkSize), + )), + ); + } + + public function test_a_remote_stream_is_read_in_steps_of_characters_read_in_line(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $stream = new LengthCapturingSourceStream("id,name\n1,a\n", path('s3://bucket/a.csv')); + + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + $stream, + (new CSVReadOptions())->withCharactersReadInLine(4096), + )); + + static::assertSame([4096], $stream->capturedIterateLengths); + } + + public function test_a_remote_stream_without_characters_read_in_line_is_read_in_default_chunks(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $stream = new LengthCapturingSourceStream("id,name\n1,a\n", path('s3://bucket/a.csv')); + + CSVFixtureContext::records(CSVFixtureContext::openNativeStream($stream)); + + static::assertSame([NativeCSVOpenSource::CHUNK], $stream->capturedIterateLengths); + } + + #[TestWith(['multiline_header.csv'])] + #[TestWith(['with_utf8_bom.csv'])] + #[TestWith(['semicolon.csv'])] + #[TestWith(['single_quotes_csv.csv'])] + public function test_columns_resolves_the_same_header_on_both_paths(string $fixture): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp($fixture); + $native = CSVFixtureContext::openNative($fixture); + + try { + static::assertSame($php->columns(), $native->columns()); + } finally { + $php->close(); + $native->close(); + } + } + + public function test_a_zero_byte_source_resolves_no_header_on_both_paths(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp('empty.csv'); + $native = CSVFixtureContext::openNative('empty.csv'); + + try { + static::assertSame([], $php->columns()); + static::assertSame([], $native->columns()); + } finally { + $php->close(); + $native->close(); + } + } + + public function test_is_supported_is_false_without_the_extension(): void + { + if (class_exists(RustCSVReaderNative::class, false)) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is loaded'); + } + + static::assertFalse(NativeCSVOpenSource::isSupported()); + } + + /** + * @param int<0, max>|-1 $rowBudget + */ + #[TestWith([2, 2])] + #[TestWith([-1, 5])] + #[TestWith([10, 5])] + public function test_sniff_folds_at_most_the_row_budget(int $rowBudget, int $rows): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('five_rows.csv'); + + try { + static::assertSame( + $rows, + $open->sniff( + ['id', 'name'], + $rowBudget, + infer_schema()->build(), + new StringTypeNarrower(InferredTypes::default()->toArray()), + )->rows(), + ); + } finally { + $open->close(); + } + } + + public function test_sniff_folds_date_and_time_zone_columns_like_observing_the_records(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $contents = "at,day,zone,n\n2024-01-01 10:00:00,2024-01-01,Europe/Warsaw,\n2024-02-02 11:00:00,2024-02-02,+02:00,1\n"; + $inference = infer_schema()->build(); + $typer = new StringTypeNarrower($inference->candidates()->toArray()); + $names = ['at', 'day', 'zone', 'n']; + $observed = (new SchemaInferrer($inference, $typer))->sniff( + $names, + CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )->records(), + -1, + ); + $sniffed = CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )->sniff($names, -1, $inference, $typer); + + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } + + public function test_columns_of_a_header_without_a_trailing_newline(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + static::assertSame( + ['id', 'name'], + CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream('id,name', path('s3://bucket/a.csv')), + )->columns(), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php similarity index 81% rename from src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php rename to src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php index 6858360697..debe8ca9b9 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php @@ -15,12 +15,12 @@ use function implode; use function iterator_to_array; -final class CSVOpenSourceTest extends FlowTestCase +final class PhpCSVOpenSourceTest extends FlowTestCase { public function test_close_closes_the_stream_once(): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $open = CSVFixtureContext::open('two_rows.csv', $counting); + $open = CSVFixtureContext::openPhp('two_rows.csv', $counting); $open->close(); @@ -29,7 +29,7 @@ public function test_close_closes_the_stream_once(): void public function test_columns_of_an_empty_source_is_empty(): void { - $open = CSVFixtureContext::open('empty.csv'); + $open = CSVFixtureContext::openPhp('empty.csv'); try { static::assertSame([], $open->columns()); @@ -40,7 +40,7 @@ public function test_columns_of_an_empty_source_is_empty(): void public function test_columns_reads_a_quoted_multiline_header_as_one_record(): void { - $open = CSVFixtureContext::open('multiline_header.csv'); + $open = CSVFixtureContext::openPhp('multiline_header.csv'); try { static::assertSame(['id', "na\nme"], $open->columns()); @@ -51,7 +51,7 @@ public function test_columns_reads_a_quoted_multiline_header_as_one_record(): vo public function test_columns_reports_the_resolved_header(): void { - $open = CSVFixtureContext::open('header_only.csv'); + $open = CSVFixtureContext::openPhp('header_only.csv'); try { static::assertSame(['id', 'name'], $open->columns()); @@ -62,7 +62,7 @@ public function test_columns_reports_the_resolved_header(): void public function test_columns_without_a_header_line_are_generated(): void { - $open = CSVFixtureContext::open('two_columns.csv', options: new CSVReadOptions(withHeader: false)); + $open = CSVFixtureContext::openPhp('two_columns.csv', options: new CSVReadOptions(withHeader: false)); try { static::assertSame(['e00', 'e01'], $open->columns()); @@ -73,7 +73,7 @@ public function test_columns_without_a_header_line_are_generated(): void public function test_records_yields_every_row_with_the_full_key_set(): void { - $open = CSVFixtureContext::open('ragged.csv'); + $open = CSVFixtureContext::openPhp('ragged.csv'); try { $records = iterator_to_array($open->records(), false); @@ -90,7 +90,7 @@ public function test_records_yields_every_row_with_the_full_key_set(): void public function test_records_yields_one_logical_record_per_iteration(): void { - $open = CSVFixtureContext::open('multiline_strings.csv'); + $open = CSVFixtureContext::openPhp('multiline_strings.csv'); try { $records = iterator_to_array($open->records(), false); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php new file mode 100644 index 0000000000..7864e1ccb3 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php @@ -0,0 +1,27 @@ +endsOutsideAnEnclosure($buffer), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php index 750e0f27ad..36d7a16fa2 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php @@ -74,6 +74,30 @@ public function test_decode_turns_empty_fields_into_null_by_default(): void static::assertSame(['id' => '1', 'name' => null], (new CSVEncoder())->decode(['id,name', '1,'])[0]->values); } + public function test_decoding_a_blank_line_yields_a_single_null_field(): void + { + static::assertSame( + ['e00' => null], + (new CSVEncoder(withHeader: false, emptyToNull: false))->decode([''])[0]->values, + ); + } + + public function test_decoding_a_line_with_a_custom_separator_enclosure_and_escape_yields_strings(): void + { + static::assertSame( + ['id' => '1', 'name' => 'a;b'], + (new CSVEncoder(separator: ';', enclosure: "'", escape: '|'))->decode(['id;name', "1;'a;b'"])[0]->values, + ); + } + + public function test_decoding_a_line_with_empty_quoted_fields_yields_empty_strings(): void + { + static::assertSame( + ['id' => '', 'name' => ''], + (new CSVEncoder(emptyToNull: false))->decode(['id,name', '"",""'])[0]->values, + ); + } + public function test_encode_renders_array_values_as_json(): void { static::assertSame( diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php index 0ce9fce2aa..ae7675d4b2 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php @@ -7,17 +7,167 @@ use Flow\ETL\Adapter\CSV\CSVLineReader; use Flow\ETL\Adapter\CSV\Tests\Double\LengthCapturingSourceStream; use Flow\Filesystem\Stream\MemorySourceStream; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use function Flow\Filesystem\DSL\path; +use function iterator_to_array; +use function preg_last_error; +use function str_repeat; + +use const PREG_NO_ERROR; final class CSVLineReaderTest extends TestCase { + /** + * @return Generator}> + */ + public static function record_boundaries(): Generator + { + yield 'ends outside an enclosure' => [',', '"', '\\', "\"a\",b\nc,d", ['"a",b', 'c,d']]; + yield 'ends inside an enclosure' => [',', '"', '\\', "\"a\nb\",c\nd", ["\"a\nb\",c", 'd']]; + yield 'escaped enclosure' => [',', '"', '\\', "\"x\\\"y\nz\",1\n\"p\",2", ["\"x\\\"y\nz\",1", '"p",2']]; + yield 'escape before the line end' => [',', '"', '\\', "\"x\\\n\",1\n\"p\",2", ["\"x\\\n\",1", '"p",2']]; + yield 'doubled enclosure' => [',', '"', '\\', "\"a\"\"\nb\",1\n\"c\",2", ["\"a\"\"\nb\",1", '"c",2']]; + yield 'doubled enclosure at the buffer end' => [',', '"', '\\', "\"a\"\"\n\",1", ["\"a\"\"\n\",1"]]; + yield 'enclosure inside an unenclosed field' => [',', '"', '\\', "x\"y,1\n\"p\",2", ['x"y,1', '"p",2']]; + yield 'blanks before an opening enclosure' => [',', '"', '\\', "a, \t\"b\nc\"\nd", ["a, \t\"b\nc\"", 'd']]; + yield 'junk after a closing enclosure' => [',', '"', '\\', "\"a\"x\"y,1\nb", ['"a"x"y,1', 'b']]; + yield 'empty escape' => [',', '"', '', "\"x\\\",1\n\"p\",2", ['"x\\",1', '"p",2']]; + yield 'escape equal to the enclosure' => [',', '"', '"', "\"a\"\"\nb\",1\nc", ["\"a\"\"\nb\",1", 'c']]; + yield 'custom separator and enclosure' => [ + ';', + "'", + '\\', + "x,'y;1\na;'b\nc';d\ne;f", + ["x,'y;1", "a;'b\nc';d", 'e;f'], + ]; + } + + public function test_a_record_ending_outside_an_enclosure_is_complete(): void + { + static::assertSame( + ['"a",b', 'c,d'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\",b\nc,d"))), + ); + } + + public function test_a_record_ending_inside_an_enclosure_is_incomplete(): void + { + static::assertSame( + ["\"a\nb\",c", 'd'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\nb\",c\nd"))), + ); + } + + public function test_an_escaped_enclosure_does_not_close_the_record(): void + { + static::assertSame( + ["\"x\\\"y\nz\",1", '"p",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"x\\\"y\nz\",1\n\"p\",2"))), + ); + } + + public function test_a_doubled_enclosure_does_not_close_the_record(): void + { + static::assertSame( + ["\"a\"\"\nb\",1", '"c",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\"\"\nb\",1\n\"c\",2"))), + ); + } + + public function test_an_enclosure_inside_an_unenclosed_field_is_a_literal_byte(): void + { + static::assertSame( + ['x"y,1', '"p",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("x\"y,1\n\"p\",2"))), + ); + } + + public function test_an_empty_escape_makes_the_escape_character_ordinary(): void + { + $content = "\"x\\\",1\n\"p\",2"; + + static::assertSame( + ["\"x\\\",1\n\"p\",2"], + iterator_to_array((new CSVLineReader('"', escape: '\\'))->readLines(new MemorySourceStream($content))), + ); + static::assertSame( + ['"x\\",1', '"p",2'], + iterator_to_array((new CSVLineReader('"', escape: ''))->readLines(new MemorySourceStream($content))), + ); + } + + public function test_a_buffer_without_any_enclosure_is_complete(): void + { + static::assertSame( + ['a,b', 'c,d'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("a,b\nc,d"))), + ); + } + + public function test_a_custom_separator_and_enclosure_are_honoured(): void + { + static::assertSame( + ["x,'y;1", "a;'b\nc';d", 'e;f'], + iterator_to_array((new CSVLineReader("'", ';'))->readLines( + new MemorySourceStream("x,'y;1\na;'b\nc';d\ne;f"), + )), + ); + } + + public function test_a_buffer_of_one_megabyte_without_a_closing_enclosure_does_not_blow_the_pcre_backtrack_limit(): void + { + $open = '"' . str_repeat('ab,\\"""c', 1 << 17); + + static::assertSame( + [$open . "\nd"], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($open . "\nd"))), + ); + static::assertSame(PREG_NO_ERROR, preg_last_error()); + } + + public function test_a_record_with_more_fields_than_pcre_can_match_is_still_split_at_its_end(): void + { + $closed = str_repeat('"a",', 200_000) . '"b"'; + $open = str_repeat('"a",', 200_000) . '"b'; + + static::assertSame( + [$closed, 'c'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($closed . "\nc"))), + ); + static::assertSame( + [$open . "\nc\""], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($open . "\nc\""))), + ); + } + + /** + * @param non-empty-string $content + * @param list $expected + */ + #[DataProvider('record_boundaries')] + public function test_the_pattern_splits_records_at_their_end( + string $separator, + string $enclosure, + string $escape, + string $content, + array $expected, + ): void { + static::assertSame( + $expected, + iterator_to_array((new CSVLineReader($enclosure, $separator, $escape))->readLines( + new MemorySourceStream($content), + )), + ); + } + public function test_characters_read_in_line_is_passed_through_to_the_stream(): void { $stream = new LengthCapturingSourceStream("id,name\n1,foo", path('s3://bucket/users.csv')); - iterator_to_array((new CSVLineReader('"', 4096))->readLines($stream)); + iterator_to_array((new CSVLineReader('"', charactersReadInLine: 4096))->readLines($stream)); static::assertSame([4096], $stream->capturedLengths); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php new file mode 100644 index 0000000000..2c4dfbea75 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php @@ -0,0 +1,34 @@ +isComplete($buffer)); + } + + public function test_a_record_with_more_fields_than_pcre_can_match_is_decided_by_the_scan(): void + { + $boundary = new CSVRecordBoundary('"'); + + static::assertTrue($boundary->isComplete(str_repeat('"a",', 200_000) . '"b"')); + static::assertFalse($boundary->isComplete(str_repeat('"a",', 200_000) . '"b')); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php new file mode 100644 index 0000000000..6ad2c73203 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php @@ -0,0 +1,87 @@ +open( + CSVFixtureContext::memorySource(), + ); + + try { + static::assertInstanceOf(NativeCSVOpenSource::class, $open); + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_open_returns_the_php_source_when_the_extension_is_absent(): void + { + if (NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is loaded'); + } + + $open = (new CSVSourceOpener(CSVFixtureContext::memory("id,name\n1,a\n"), new CSVReadOptions()))->open( + CSVFixtureContext::memorySource(), + ); + + try { + static::assertInstanceOf(PhpCSVOpenSource::class, $open); + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_a_failing_detection_closes_the_stream_on_both_paths(): void + { + $throwing = new ThrowingSourceFilesystem(CSVFixtureContext::memory("id,name\n1,a\n")); + + $this->expectException(RuntimeException::class); + + try { + (new CSVSourceOpener($throwing, new CSVReadOptions()))->open(CSVFixtureContext::memorySource()); + } finally { + static::assertTrue($throwing->lastStream?->closed, 'open() closes the stream it opened before rethrowing'); + } + } + + public function test_characters_read_in_line_reaches_the_opened_source(): void + { + $filesystem = new LengthCapturingFilesystem(CSVFixtureContext::memory("id,name\n1,a\n")); + $open = (new CSVSourceOpener($filesystem, (new CSVReadOptions())->withCharactersReadInLine(4096)))->open( + CSVFixtureContext::memorySource(), + ); + + try { + iterator_to_array($open->records()); + } finally { + $open->close(); + } + + NativeCSVOpenSource::isSupported() + ? static::assertSame([4096], $filesystem->lastStream?->capturedIterateLengths) + : static::assertSame([null, 4096], $filesystem->lastStream?->capturedLengths); + } +} diff --git a/src/core/etl/src/Flow/ETL/Row.php b/src/core/etl/src/Flow/ETL/Row.php index 3c3f1b6bea..8977093468 100644 --- a/src/core/etl/src/Flow/ETL/Row.php +++ b/src/core/etl/src/Flow/ETL/Row.php @@ -165,6 +165,21 @@ public function values(): array private function conform(Schema $schema, bool $checkValues): self { $definitions = $schema->definitions(); + + if (count($this->values) === count($definitions) && array_keys($this->values) === array_keys($definitions)) { + foreach ($definitions as $name => $definition) { + if ( + $checkValues + ? !$definition->matches($this->values[$name]) + : $this->values[$name] === null && !$definition->isNullable() + ) { + throw ColumnMismatchException::valueDoesNotMatch($definition, $this->values[$name]); + } + } + + return $this; + } + $matched = []; $taken = 0; diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php index a1a8154de3..fee2112843 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php @@ -7,7 +7,9 @@ use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; use Flow\Types\Type; +use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Native\NullType; +use Flow\Types\Type\Native\StringType; use Flow\Types\Type\TypeNarrower; use Flow\Types\Type\TypeWidener; @@ -44,6 +46,24 @@ public function __construct( } } + /** + * A fold computed elsewhere (a native reader, a subprocess) - equal to the observe() fold over the same rows. + * + * @param array> $types - first-seen order, header names first + */ + public static function fromColumnTypes( + array $types, + int $rows, + TypeNarrower $typer, + TypeWidener $widener = new TypeWidener(), + ): self { + $columns = new self([], $typer, $widener); + $columns->types = $types; + $columns->rows = $rows; + + return $columns; + } + /** * Merge left-to-right in listing order: name order follows first-seen, so a different bracketing of the name * sequence changes the definition order (never the types). @@ -78,6 +98,19 @@ public function observe(RawRowValues $row): void { /** @var mixed $value */ foreach ($row->values as $name => $value) { + $current = array_key_exists($name, $this->types) ? $this->types[$name] : null; + + if ( + $value !== null + && ( + $current instanceof StringType + || $current instanceof OptionalType + && $current->base() instanceof StringType + ) + ) { + continue; + } + $observed = match (true) { $value === null => type_null(), is_string($value) && trim($value) !== $value => type_string(), @@ -88,9 +121,7 @@ public function observe(RawRowValues $row): void $observed = type_string(); } - $this->types[$name] = array_key_exists($name, $this->types) - ? $this->widener->widen($this->types[$name], $observed) - : $observed; + $this->types[$name] = $current === null ? $observed : $this->widener->widen($current, $observed); } $this->rows++; diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php index 12e9383b06..9bc005ea82 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php @@ -40,11 +40,15 @@ public function infer(array $names, iterable $sources): Schema break; } - $partial = $this->sniff( - $names, - $source, - $this->inference->sampleSize === -1 ? -1 : max(0, $this->inference->sampleSize - $columns->rows()), - ); + $rowBudget = $this->inference->sampleSize === -1 + ? -1 + : max(0, $this->inference->sampleSize - $columns->rows()); + + if ($source instanceof SniffsColumnTypes) { + $partial = $source->sniffColumnTypes($names, $rowBudget, $this->inference, $this->typer); + } else { + $partial = $this->sniff($names, $source, $rowBudget); + } if ($partial->rows() > 0) { $sniffed++; diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php index 567c2c9ff9..d7224f93c1 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php @@ -10,9 +10,8 @@ interface SchemaSampler { /** * One inner iterable per sampling unit (a file, a byte-range chunk, a sheet range), in listing order. - * The outer iterable is exactly what SchemaInferrer::infer() consumes; a future parallel driver fans - * SchemaInferrer::sniff() over the inner iterables and merges the ColumnTypes partials. - * The unit itself is never named as a type - the implementer decides what one inner iterable is. + * The outer iterable is exactly what SchemaInferrer::infer() consumes; a unit that implements + * SniffsColumnTypes folds itself into its partial instead of being iterated row by row. * * @return iterable> */ diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php new file mode 100644 index 0000000000..97b55c0a3d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php @@ -0,0 +1,28 @@ + + */ +interface SniffsColumnTypes extends Traversable +{ + /** + * Must equal SchemaInferrer::sniff($names, , $rowBudget). + * + * @param list $names - columns known before any row + * @param int<0, max>|-1 $rowBudget - rows to observe, 0 for none, -1 for all of them + */ + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes; +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php new file mode 100644 index 0000000000..f63085dfce --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php @@ -0,0 +1,51 @@ + + */ +final class SpySniffingSample implements IteratorAggregate, SniffsColumnTypes +{ + /** + * @var list, int}> + */ + public array $sniffed = []; + + public function __construct( + private readonly ColumnTypes $fold, + ) {} + + /** + * @return ArrayIterator + */ + public function getIterator(): ArrayIterator + { + throw new RuntimeException('a sample that sniffs itself is never iterated'); + } + + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes { + $this->sniffed[] = [$names, $rowBudget]; + + return $this->fold; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php index 9b3d51eaab..e8e25a9228 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php @@ -7,6 +7,7 @@ use Flow\ETL\Schema\Inference\ColumnTypes; use Flow\ETL\Schema\Inference\InferredTypes; use Flow\ETL\Schema\Inference\TypeFloor; +use Flow\Types\Type; use Flow\Types\Type\Logical\InstanceOfTypeNarrower; use Flow\Types\Type\Native\String\StringTypeNarrower; @@ -24,6 +25,16 @@ public static function floor(): TypeFloor return new TypeFloor(InferredTypes::default()); } + /** + * A fold computed elsewhere - a native reader, a subprocess. + * + * @param array> $types + */ + public static function fromColumnTypes(array $types, int $rows): ColumnTypes + { + return ColumnTypes::fromColumnTypes($types, $rows, new StringTypeNarrower(InferredTypes::default()->toArray())); + } + /** * @param list $names */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php index 5407bca89d..f93ee1d54c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php @@ -26,6 +26,31 @@ final class RowTest extends FlowTestCase { + public function test_conform_to_a_schema_in_a_different_order_reorders_the_values(): void + { + static::assertSame( + ['b' => 2, 'a' => 1], + row(['a' => 1, 'b' => 2])->conformTo(schema(int_schema('b'), int_schema('a')))->values(), + ); + } + + public function test_conform_to_a_schema_the_row_already_matches_returns_an_equal_row(): void + { + $row = row(['a' => 1, 'b' => 'x']); + + static::assertEquals($row, $row->conformTo(schema(int_schema('a'), str_schema('b')))); + } + + public function test_conform_to_a_schema_with_as_many_but_other_columns_refuses_the_row(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'Row does not match its schema: column "b" declared by the schema is missing from the row', + ); + + row(['a' => 1, 'z' => 2])->conformTo(schema(int_schema('a'), int_schema('b'))); + } + public function test_conform_to_does_not_validate_a_non_null_value(): void { // the caller produced the value by casting to the column's type - conformTo() checks the shape only @@ -154,6 +179,16 @@ public function test_match_to_reports_no_row_coordinate_of_its_own(): void row(['a' => 1])->matchTo(schema()); } + public function test_match_to_with_the_same_keys_still_validates_every_value(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'Row does not match its schema: column "c": could not convert 3 (integer) to string', + ); + + row(['a' => 1, 'b' => 'x', 'c' => 3])->matchTo(schema(int_schema('a'), str_schema('b'), str_schema('c'))); + } + public function test_get_throws_when_the_column_is_absent(): void { $this->expectException(InvalidArgumentException::class); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php index fbcd5480a7..1fe18e5146 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php @@ -150,6 +150,10 @@ public static function widenedColumns(): array 'integer then text' => [['1', 'x'], type_string()], 'date then datetime' => [['2024-01-01', '2024-01-01 10:00'], type_datetime()], 'both booleans' => [['true', 'false'], type_boolean()], + 'free text then datetime' => [['free text', '2024-01-01 10:00:00'], type_string()], + 'free text then integer' => [['free text', '42'], type_string()], + 'free text then json' => [['free text', '{"a":1}'], type_string()], + 'free text then uuid' => [['free text', 'f47ac10b-58cc-4372-a567-0e02b2c3d479'], type_string()], ]; } @@ -174,6 +178,19 @@ public function test_a_name_met_only_in_a_row_is_appended_after_the_header_names static::assertSame(['a', 'b', 'c'], array_keys($columns->schema(ColumnTypesMother::floor())->definitions())); } + public function test_a_null_first_row_then_free_text_yields_optional_string(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => 'free text'])); + $columns->observe(new RawRowValues(['c' => '42'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_a_numeric_key_becomes_a_string_column_name(): void { $columns = ColumnTypesMother::fromStrings(); @@ -184,6 +201,19 @@ public function test_a_numeric_key_becomes_a_string_column_name(): void static::assertSame('1', $columns->schema(ColumnTypesMother::floor())->get('1')->entry()->name()); } + public function test_a_saturated_string_column_widens_to_optional_string_on_a_later_null(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => 'free text'])); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => 'more text'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_a_source_with_no_names_and_no_rows_yields_an_empty_schema(): void { static::assertEquals(new Schema(), ColumnTypesMother::fromStrings()->schema(ColumnTypesMother::floor())); @@ -231,6 +261,64 @@ public function test_an_integer_column_is_declared_nullable(): void static::assertTrue($definition->isNullable()); } + public function test_an_optional_non_string_column_is_not_saturated(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => '1'])); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => '1.5'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_float()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + + public function test_from_column_types_reproduces_an_observed_fold(): void + { + $observed = ColumnTypesMother::fromStrings(['id', 'name']); + $observed->observe(new RawRowValues(['id' => '1', 'name' => 'a'])); + $observed->observe(new RawRowValues(['id' => '2', 'name' => null])); + + $computed = ColumnTypesMother::fromColumnTypes([ + 'id' => type_optional(type_integer()), + 'name' => type_optional(type_string()), + ], 2); + + static::assertEquals( + $observed->schema(ColumnTypesMother::floor()), + $computed->schema(ColumnTypesMother::floor()), + ); + static::assertSame($observed->rows(), $computed->rows()); + } + + public function test_from_column_types_merges_with_an_observed_fold(): void + { + $left = ColumnTypesMother::fromStrings(['a']); + $left->observe(new RawRowValues(['a' => '1'])); + $right = ColumnTypesMother::fromStrings(['a']); + $right->observe(new RawRowValues(['a' => '1.5', 'b' => 'x'])); + + static::assertEquals( + $left->merge($right, true)->schema(ColumnTypesMother::floor()), + $left->merge(ColumnTypesMother::fromColumnTypes([ + 'a' => type_optional(type_float()), + 'b' => type_string(), + ], 1), true)->schema(ColumnTypesMother::floor()), + ); + } + + public function test_from_column_types_carries_the_row_count_into_merge(): void + { + $observed = ColumnTypesMother::fromStrings(); + $observed->observe(new RawRowValues(['a' => '1'])); + + static::assertSame( + 8, + $observed->merge(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 7), true)->rows(), + ); + } + public function test_merge_is_associative_over_three_partials(): void { $left = ColumnTypesMother::fromStrings(['a']); @@ -383,6 +471,21 @@ public function test_null_never_reaches_the_ladder(): void ); } + public function test_observing_a_column_name_absent_from_the_header_still_records_its_type(): void + { + $columns = ColumnTypesMother::fromStrings(['a']); + $columns->observe(new RawRowValues(['a' => 'free text', 'b' => '1'])); + $columns->observe(new RawRowValues(['a' => 'more text', 'b' => '2'])); + + static::assertEquals( + new Schema( + definition_from_type('a', type_string(), nullable: true), + definition_from_type('b', type_integer(), nullable: true), + ), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_padded_cells_are_read_as_text_because_cast_does_not_trim(): void { $padded = ColumnTypesMother::fromStrings(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php index 256e4ef765..b6b1a732b1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Schema\Inference\SchemaInferrer; use Flow\ETL\Tests\Double\FakeSchemaSampler; use Flow\ETL\Tests\Double\RecordingSources; +use Flow\ETL\Tests\Double\SpySniffingSample; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\ColumnTypesMother; use Flow\Types\Type\Native\String\StringTypeNarrower; @@ -19,6 +20,8 @@ use function array_keys; use function array_slice; use function Flow\ETL\DSL\definition_from_type; +use function Flow\Types\DSL\type_date; +use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; @@ -77,6 +80,70 @@ public static function threeSourcesOfTenRows(): array return $sources; } + public function test_a_sample_that_sniffs_itself_is_folded_with_the_budget_left(): void + { + $first = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 4)); + $second = new SpySniffingSample(ColumnTypesMother::fromColumnTypes([ + 'a' => type_float(), + 'b' => type_string(), + ], 3)); + + $schema = (new SchemaInferrer( + new SchemaInference(sampleSize: 10, unionByName: true), + new StringTypeNarrower(), + ))->infer(['a'], [$first, $second]); + + static::assertSame([[['a'], 10]], $first->sniffed); + static::assertSame([[['a'], 6]], $second->sniffed); + static::assertEquals( + new Schema( + definition_from_type('a', type_float(), nullable: true), + definition_from_type('b', type_string(), nullable: true), + ), + $schema, + ); + } + + public function test_a_sample_that_sniffs_itself_is_not_asked_once_the_budget_is_spent(): void + { + $first = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 10)); + $second = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 10)); + + (new SchemaInferrer(new SchemaInference(sampleSize: 10), new StringTypeNarrower()))->infer(['a'], [ + $first, + $second, + ]); + + static::assertSame([], $second->sniffed); + } + + public function test_a_column_that_is_free_text_in_the_first_source_and_numeric_in_the_second_widens_to_string(): void + { + $fixture = [ + [new RawRowValues(['id' => '1', 'v' => 'free text']), new RawRowValues(['id' => '2', 'v' => 'more text'])], + [new RawRowValues(['id' => '3', 'v' => '42']), new RawRowValues(['id' => '4', 'v' => '43'])], + ]; + $inferrer = new SchemaInferrer( + new SchemaInference(), + new StringTypeNarrower(InferredTypes::default()->toArray()), + ); + + static::assertEquals( + new Schema( + definition_from_type('id', type_integer(), nullable: true), + definition_from_type('v', type_string(), nullable: true), + ), + $inferrer->infer(['id', 'v'], (new RecordingSources($fixture))->sources()), + ); + static::assertEquals( + definition_from_type('v', type_integer(), nullable: true), + $inferrer + ->sniff(['id', 'v'], (new RecordingSources($fixture))->source(1), -1) + ->schema(ColumnTypesMother::floor()) + ->get('v'), + ); + } + public function test_a_row_less_source_does_not_spend_a_files_to_sniff_slot(): void { $sources = new RecordingSources([ @@ -319,6 +386,26 @@ public function test_the_candidate_set_reaches_the_floor_through_infer(): void ); } + public function test_union_by_name_over_sources_where_one_column_saturates_to_string(): void + { + $sources = new RecordingSources([ + [new RawRowValues(['id' => '1', 'v' => 'free text'])], + [new RawRowValues(['id' => '2', 'v' => '42', 'w' => '2024-01-01'])], + ]); + + static::assertEquals( + new Schema( + definition_from_type('id', type_integer(), nullable: true), + definition_from_type('v', type_string(), nullable: true), + definition_from_type('w', type_date(), nullable: true), + ), + (new SchemaInferrer( + new SchemaInference(unionByName: true), + new StringTypeNarrower(InferredTypes::default()->toArray()), + ))->infer([], $sources->sources()), + ); + } + public function test_unbounded_bounds_read_every_row_of_every_source(): void { $sources = new RecordingSources(self::threeSourcesOfTenRows()); diff --git a/src/extension/arrow-ext/Makefile b/src/extension/arrow-ext/Makefile index fea33f1527..b8bdfd6eac 100644 --- a/src/extension/arrow-ext/Makefile +++ b/src/extension/arrow-ext/Makefile @@ -39,15 +39,15 @@ test: build sed -n '/^--FILE--$$/,/^--EXPECT/p' "$$f" | sed '1d;$$d' > "$$tmp"; \ expected=$$(sed -n '/^--EXPECT\(F\)\{0,1\}--$$/,$$p' "$$f" | sed '1d' | tr -d '\r'); \ actual=$$($(PHP) -n -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ - actual=$$(echo "$$actual" | tr -d '\r'); \ + actual=$$(printf '%s' "$$actual" | tr -d '\r'); \ rm -f "$$tmp"; \ if [ "$$actual" = "$$expected" ]; then \ echo "PASS: $$test_name"; \ passed=$$((passed + 1)); \ else \ echo "FAIL: $$test_name"; \ - echo " Expected: $$expected"; \ - echo " Actual: $$actual"; \ + printf ' Expected: %s\n' "$$expected"; \ + printf ' Actual: %s\n' "$$actual"; \ failed=1; \ fi; \ done; \ diff --git a/src/extension/flow-php-ext/Cargo.lock b/src/extension/flow-php-ext/Cargo.lock index 0d97510639..2dd3b5705b 100644 --- a/src/extension/flow-php-ext/Cargo.lock +++ b/src/extension/flow-php-ext/Cargo.lock @@ -446,6 +446,7 @@ name = "flow_php" version = "0.0.0" dependencies = [ "ext-php-rs", + "memchr", "serde", "serde_json", ] diff --git a/src/extension/flow-php-ext/Cargo.toml b/src/extension/flow-php-ext/Cargo.toml index b7ed1188bb..0eb36b00b0 100644 --- a/src/extension/flow-php-ext/Cargo.toml +++ b/src/extension/flow-php-ext/Cargo.toml @@ -8,5 +8,6 @@ crate-type = ["cdylib"] [dependencies] ext-php-rs = "0.15" +memchr = "2" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["raw_value"] } diff --git a/src/extension/flow-php-ext/Makefile b/src/extension/flow-php-ext/Makefile index 5cdd7fb8af..f307e63ba1 100644 --- a/src/extension/flow-php-ext/Makefile +++ b/src/extension/flow-php-ext/Makefile @@ -51,15 +51,15 @@ test: build sed -n '/^--FILE--$$/,/^--EXPECT/p' "$$f" | sed '1d;$$d' > "$$tmp"; \ expected=$$(sed -n '/^--EXPECT\(F\)\{0,1\}--$$/,$$p' "$$f" | sed '1d' | tr -d '\r'); \ actual=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ - actual=$$(echo "$$actual" | tr -d '\r'); \ + actual=$$(printf '%s' "$$actual" | tr -d '\r'); \ rm -f "$$tmp"; \ if [ "$$actual" = "$$expected" ]; then \ echo "PASS: $$test_name"; \ passed=$$((passed + 1)); \ else \ echo "FAIL: $$test_name"; \ - echo " Expected: $$expected"; \ - echo " Actual: $$actual"; \ + printf ' Expected: %s\n' "$$expected"; \ + printf ' Actual: %s\n' "$$actual"; \ failed=1; \ fi; \ done; \ diff --git a/src/extension/flow-php-ext/composer.json b/src/extension/flow-php-ext/composer.json index e78a7ed00f..c21e562074 100644 --- a/src/extension/flow-php-ext/composer.json +++ b/src/extension/flow-php-ext/composer.json @@ -9,6 +9,7 @@ }, "autoload": { "psr-4": { + "Flow\\ETL\\Adapter\\CSV\\": "php/Flow/ETL/Adapter/CSV/", "Flow\\ETL\\Row\\": "php/Flow/ETL/Row/", "Flow\\Floe\\": "php/Flow/Floe/" } diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php new file mode 100644 index 0000000000..aac509ac5c --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php @@ -0,0 +1,66 @@ +|-1 $limit + * + * @return int<0, max> + */ + public function fold(RustColumnFoldNative $fold, int $limit): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return list + */ + public function headers(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @param int<1, max> $batchSize + * + * @return list + */ + public function next(int $batchSize): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php new file mode 100644 index 0000000000..57e98214ee --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php @@ -0,0 +1,46 @@ + $names + * @param list $candidates - the candidate types' toString() + */ + public function __construct(array $names, array $candidates) + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function narrowOne(string $value): string + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return int<0, max> + */ + public function rows(): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return array + */ + public function types(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/src/cast.rs b/src/extension/flow-php-ext/src/cast.rs index 3d55bb25bb..7a849804bd 100644 --- a/src/extension/flow-php-ext/src/cast.rs +++ b/src/extension/flow-php-ext/src/cast.rs @@ -9,11 +9,12 @@ use ext_php_rs::zend::Function; use crate::ctx::{ array_key_index, call_handle, call_handle_catching, call_handle_on, call_handle_transparent, ce_method_ref, construct_with_zvals, ht_find_key, ht_insert, ht_insert_key, null_zval, - schema_mismatch, transparent_exception, write_slot, zval_long, zval_str, Ctx, HtKey, + schema_mismatch, transparent_exception, write_slot, zval_long, Ctx, HtKey, }; use crate::encode::{expect_object, ht_for_each, read_slot}; use crate::exception::ext_exception; use crate::hydrate::{build_hydrate_plan, fold_metadata_into_schema, AssemblyClasses, HydratePlan, RowValuesClass}; +use crate::json_check::json_valid; use crate::plan::{parse_schema_json, TypeJson}; use crate::values::date_from_free_form; @@ -326,13 +327,92 @@ fn json_gate(bytes: &[u8]) -> bool { || (bytes[0] == b'[' && bytes[bytes.len() - 1] == b']')) } -fn json_object_from(bytes: &[u8], ctx: &mut Ctx) -> Result { +/// `DateTimeType::ISO_DATE_TIME` followed by `checkdate()`: true only when PHP takes its +/// `new DateTimeImmutable($value)` branch without consulting StringTemporalParts. +fn iso_date_time_gate(bytes: &[u8]) -> bool { + // PCRE `$` without the D modifier also matches before one final "\n" + let bytes = bytes.strip_suffix(b"\n").unwrap_or(bytes); + let byte_at = |at: usize| bytes.get(at).copied(); + let digits_at = |at: usize, count: usize| { + bytes + .get(at..at + count) + .is_some_and(|run| run.iter().all(u8::is_ascii_digit)) + }; + + if !(digits_at(0, 4) + && byte_at(4) == Some(b'-') + && digits_at(5, 2) + && byte_at(7) == Some(b'-') + && digits_at(8, 2) + && matches!(byte_at(10), Some(b'T' | b' ')) + && digits_at(11, 2) + && byte_at(13) == Some(b':') + && digits_at(14, 2)) + { + return false; + } + + let mut at = 16; + + if byte_at(at) == Some(b':') && digits_at(at + 1, 2) { + at += 3; + + if byte_at(at) == Some(b'.') { + let fraction = bytes[at + 1..].iter().take_while(|byte| byte.is_ascii_digit()).count(); + + if !(1..=9).contains(&fraction) { + return false; + } + + at += 1 + fraction; + } + } + + match byte_at(at) { + Some(b'Z') => at += 1, + Some(b'+' | b'-') if digits_at(at + 1, 2) => { + at += 3; + + if byte_at(at) == Some(b':') && digits_at(at + 1, 2) { + at += 3; + } else if digits_at(at, 2) { + at += 2; + } + } + _ => {} + } + + let number = |range: std::ops::Range| { + bytes[range] + .iter() + .fold(0u32, |value, digit| value * 10 + u32::from(digit - b'0')) + }; + let (year, month, day) = (number(0..4), number(5..7), number(8..10)); + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let days_in_month = match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + + at == bytes.len() && year >= 1 && (1..=12).contains(&month) && (1..=days_in_month).contains(&day) +} + +/// `source` is a gated JSON string; the Json shares its zend_string instead of copying the bytes. +fn json_object_from(source: &Zval, ctx: &mut Ctx) -> Result { + let bytes = source + .zend_str() + .ok_or_else(|| ext_exception("flow_php expected a JSON string"))? + .as_bytes(); + let is_object = bytes[0] == b'{' && bytes[bytes.len() - 1] == b'}'; + let (json_ce, value_slot, is_object_slot) = ctx.json()?; let mut json = ZendObject::new(json_ce); - write_slot(&mut json, value_slot, zval_str(bytes)); + write_slot(&mut json, value_slot, source.shallow_clone()); let mut is_object_zv = Zval::new(); - is_object_zv.set_bool(bytes[0] == b'{' && bytes[bytes.len() - 1] == b'}'); + is_object_zv.set_bool(is_object); write_slot(&mut json, is_object_slot, is_object_zv); let mut zv = Zval::new(); @@ -411,6 +491,10 @@ fn numeric_scalar(value: &Zval) -> bool { /// PHP calls made inside a branch discard their own thrown exceptions and bail /// instead - mirroring the `catch (Throwable)` wrappers in the PHP casts. fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result, PhpException> { + // Type::cast receives its argument by value; a branch that returns `value` itself must not hand + // back the caller's reference + let value = value.dereference(); + Ok(match kind { CastKind::Integer => { if value.is_long() { @@ -507,10 +591,16 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result Result Result, PhpException> } if value.is_string() { - // see CastKind::DateTime above - the string gate lives in PHP + // DateType::cast has no ISO branch: every string is gated on StringTemporalParts, a PHP class return Ok(None); } @@ -736,18 +826,12 @@ fn cast_json(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> return Ok(None); } - let json_validate = ctx.json_validate()?; - let Ok(valid) = - call_handle_transparent(json_validate, None, &mut [value.shallow_clone()]) - else { - return Ok(None); - }; - - if !valid.bool().unwrap_or(false) { + if !json_valid(string.as_bytes()) { + // a reject is never authoritative: the retained PHP Type::cast re-asks json_validate() return Ok(None); } - return Ok(Some(json_object_from(string.as_bytes(), ctx)?)); + return Ok(Some(json_object_from(value, ctx)?)); } if value.is_array() { @@ -767,7 +851,7 @@ fn cast_json(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> return Ok(None); } - return Ok(Some(json_object_from(string.as_bytes(), ctx)?)); + return Ok(Some(json_object_from(&encoded, ctx)?)); } Ok(None) @@ -795,6 +879,7 @@ pub fn cast_rows( let schema_zv = fold_metadata_into_schema(schema, batch_ht, raw_class, ctx)?; let mut rows_ht = ZendHashTable::with_capacity(batch_ht.len() as u32); + let mut every_column_present = true; ht_for_each(batch_ht, |_, row_index, rv_zv| { let rv = expect_object(rv_zv, "a RawRowValues")?; @@ -813,7 +898,8 @@ pub fn cast_rows( // refuse the absence mid-row, so a later cast refusal in the same batch would be // reported by PHP and pre-empted by native - the two paths would name different // columns and different rows for the same input. - let Some(value) = ht_find_key(values_ht, &key) else { + let Some(value) = ht_find_key(values_ht, &key).map(Zval::dereference) else { + every_column_present = false; continue; }; @@ -885,6 +971,12 @@ pub fn cast_rows( rows_zv.set_hashtable(rows_ht); // every non-null value was just cast to its column's type, so the batch takes - // the shape-only door - the one HydratedBatch returns through - call_handle(assembly.rows_conformed, None, &mut [schema_zv, rows_zv], "conform Rows") + // the shape-only door - the one HydratedBatch returns through. Row::conform() returns + // $this for a row holding exactly the Schema's keys, in order, with no null under + // NOT NULL - which every row built above is once no column was absent. An absence + // keeps the PHP door: padding, missingColumn and the failing row's index are + // conform()'s to decide. + let door = if every_column_present { assembly.rows_trusted } else { assembly.rows_conformed }; + + call_handle(door, None, &mut [schema_zv, rows_zv], "assemble Rows") } diff --git a/src/extension/flow-php-ext/src/csv/fold.rs b/src/extension/flow-php-ext/src/csv/fold.rs new file mode 100644 index 0000000000..d963b41319 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/fold.rs @@ -0,0 +1,453 @@ +//! Schema inference over CSV cells, exactly as `ColumnTypes::observe()` + `StringTypeNarrower::narrow()` + +//! `TypeWidener::widen()` fold them, for the closed set of types a CSV cell can narrow to. `date_parse()` and +//! `new DateTimeZone('+HH:MM')` are called back into PHP, each behind a Rust pre-filter that is a necessary condition +//! of the PHP predicate, so parity on those rungs holds by construction. A JSON cell is accepted natively and only a +//! native reject is re-asked through `json_validate()`. + +use std::collections::HashMap; + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::ctx::{call_handle, zval_str, Ctx}; +use crate::csv::php_trim; +use crate::csv::tokenizer::is_space; +use crate::exception::ext_exception; +use crate::json_check::json_valid; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Leaf { + Null, + String, + Json, + Uuid, + Float, + Integer, + DateTime, + Date, + Boolean, + TimeZone, +} + +impl Leaf { + pub fn code(self) -> &'static str { + match self { + Leaf::Null => "null", + Leaf::String => "string", + Leaf::Json => "json", + Leaf::Uuid => "uuid", + Leaf::Float => "float", + Leaf::Integer => "integer", + Leaf::DateTime => "datetime", + Leaf::Date => "date", + Leaf::Boolean => "boolean", + Leaf::TimeZone => "timezone", + } + } +} + +/// A leaf, optionally nullable; `Null` itself is never optional. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Kind { + leaf: Leaf, + optional: bool, +} + +impl Kind { + fn of(leaf: Leaf) -> Self { + Self { leaf, optional: false } + } + + pub fn code(self) -> String { + if self.optional { + format!("?{}", self.leaf.code()) + } else { + self.leaf.code().to_string() + } + } + + fn is_string(self) -> bool { + self.leaf == Leaf::String + } +} + +/// `TypeWidener::widen()` over the CSV leaves; pinned against it pair by pair (`048_csv_widen_parity.phpt`). +fn widen(left: Kind, right: Kind) -> Kind { + if left == right { + return left; + } + + let optional = left.optional || right.optional || left.leaf == Leaf::Null || right.leaf == Leaf::Null; + + let leaf = match (left.leaf, right.leaf) { + (Leaf::Null, leaf) | (leaf, Leaf::Null) => leaf, + (a, b) if a == b => a, + (Leaf::Integer | Leaf::Float, Leaf::Integer | Leaf::Float) => Leaf::Float, + (Leaf::Date | Leaf::DateTime, Leaf::Date | Leaf::DateTime) => Leaf::DateTime, + _ => Leaf::String, + }; + + Kind { leaf, optional } +} + +/// The rungs `StringTypeNarrower::emitsClass()` lets run, by the candidate types' `toString()`. +struct Candidates { + json: bool, + uuid: bool, + float: bool, + integer: bool, + datetime: bool, + date: bool, + boolean: bool, + timezone: bool, +} + +impl Candidates { + /// The fold has no HTML or XML rung, so a candidate set that allows either cannot be folded natively. + fn from_codes(codes: &[Vec]) -> Result { + let has = |code: &[u8]| codes.iter().any(|candidate| candidate == code); + + if has(b"html") || has(b"xml") { + return Err(ext_exception("flow_php cannot fold html or xml candidates natively")); + } + + Ok(Self { + json: has(b"json"), + uuid: has(b"uuid"), + float: has(b"float"), + integer: has(b"integer"), + datetime: has(b"datetime"), + date: has(b"date"), + boolean: has(b"boolean"), + timezone: has(b"timezone"), + }) + } +} + +/// `StringTypeNarrower::narrow()` for a string cell, without the HTML and XML rungs. +pub struct Narrower { + candidates: Candidates, + ctx: Ctx, +} + +impl Narrower { + fn new(candidates: Candidates) -> Result { + Ok(Self { + candidates, + ctx: Ctx::new()?, + }) + } + + pub fn narrow(&mut self, value: &[u8]) -> Result { + let value = php_trim(value); + + if value.is_empty() { + return Ok(Leaf::String); + } + + if is_null(value) { + return Ok(Leaf::Null); + } + + if self.candidates.json && is_json_shaped(value) && self.is_json(value)? { + return Ok(Leaf::Json); + } + + if self.candidates.uuid && is_uuid(value) { + return Ok(Leaf::Uuid); + } + + if self.candidates.float && is_float(value) { + return Ok(Leaf::Float); + } + + if self.candidates.integer && is_integer(value) { + return Ok(Leaf::Integer); + } + + if (self.candidates.datetime || self.candidates.date) && has_explicit_day(value) { + match self.temporal(value)? { + Some(Leaf::DateTime) if self.candidates.datetime => return Ok(Leaf::DateTime), + Some(Leaf::Date) if self.candidates.date => return Ok(Leaf::Date), + _ => {} + } + } + + if self.candidates.boolean && (value.eq_ignore_ascii_case(b"true") || value.eq_ignore_ascii_case(b"false")) { + return Ok(Leaf::Boolean); + } + + if self.candidates.timezone && self.is_timezone(value)? { + return Ok(Leaf::TimeZone); + } + + Ok(Leaf::String) + } + + fn is_json(&mut self, value: &[u8]) -> Result { + if json_valid(value) { + return Ok(true); + } + + // same one-sided rule as the cast: only PHP may say "not JSON" + let valid = call_handle(self.ctx.json_validate()?, None, &mut [zval_str(value)], "validate a JSON cell")?; + + Ok(valid.bool().unwrap_or(false)) + } + + /// `StringTemporalParts::from()` past its `hasExplicitDay()` gate: `Date`, `DateTime`, or `None` when the value + /// is not a calendar date. + fn temporal(&mut self, value: &[u8]) -> Result, PhpException> { + let parts_zv = call_handle(self.ctx.date_parse()?, None, &mut [zval_str(value)], "parse a temporal cell")?; + let parts = parts_zv + .array() + .ok_or_else(|| ext_exception("flow_php expected date_parse() to return an array"))?; + + let long = |key: &str| parts.get(key).and_then(Zval::long); + let present = |key: &str| parts.get(key).is_some_and(|zv| !zv.is_false()); + + if long("error_count").unwrap_or(0) > 0 { + return Ok(None); + } + + let (Some(year), Some(month), Some(day)) = (long("year"), long("month"), long("day")) else { + return Ok(None); + }; + + if !checkdate(month, day, year) { + return Ok(None); + } + + let mut time = present("hour") || present("minute") || present("second") || present("fraction"); + + if !time { + if let Some(relative) = parts.get("relative").and_then(Zval::array) { + time = ["hour", "minute", "second"] + .iter() + .any(|key| relative.get(*key).is_some_and(|zv| !is_zero(zv))); + } + } + + Ok(Some(if time { Leaf::DateTime } else { Leaf::Date })) + } + + fn is_timezone(&mut self, value: &[u8]) -> Result { + if self.ctx.timezone_identifiers()?.contains(value) { + return Ok(true); + } + + Ok(is_offset(value) && self.ctx.timezone_accepts(value)?) + } +} + +fn is_zero(zv: &Zval) -> bool { + zv.long() == Some(0) +} + +/// `in_array(mb_strtolower($value), ['null', 'nil'], true)`: no non-ASCII codepoint lowercases into these letters. +fn is_null(value: &[u8]) -> bool { + value.is_ascii() && (value.eq_ignore_ascii_case(b"null") || value.eq_ignore_ascii_case(b"nil")) +} + +/// `Json::isValid()`'s shape gate before `json_validate()`. +fn is_json_shaped(value: &[u8]) -> bool { + matches!((value.first(), value.last()), (Some(b'{'), Some(b'}')) | (Some(b'['), Some(b']'))) +} + +/// `Uuid::isValid()`: 36 bytes, `/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/`. +fn is_uuid(value: &[u8]) -> bool { + value.len() == 36 + && value.iter().enumerate().all(|(index, byte)| match index { + 8 | 13 | 18 | 23 => *byte == b'-', + _ => byte.is_ascii_digit() || (b'a'..=b'f').contains(byte), + }) +} + +/// PHP 8 `is_numeric()` on a string: optional leading and trailing whitespace, optional sign, +/// `digits[.digits] | .digits | digits.`, optional exponent. No hex, no `INF`/`NAN`. +fn is_numeric(value: &[u8]) -> bool { + let mut position = value.iter().take_while(|byte| is_space(**byte)).count(); + let end = value.len() - value[position..].iter().rev().take_while(|byte| is_space(**byte)).count(); + + if matches!(value.get(position), Some(b'+' | b'-')) { + position += 1; + } + + let digits = |from: usize| value[from..end].iter().take_while(|byte| byte.is_ascii_digit()).count(); + + let integral = digits(position); + position += integral; + + let mut fractional = 0; + + if value.get(position) == Some(&b'.') && position < end { + fractional = digits(position + 1); + position += 1 + fractional; + } + + if integral == 0 && fractional == 0 { + return false; + } + + if position < end && matches!(value[position], b'e' | b'E') { + let mut exponent = position + 1; + + if exponent < end && matches!(value[exponent], b'+' | b'-') { + exponent += 1; + } + + let exponent_digits = digits(exponent); + + if exponent_digits == 0 { + return false; + } + + position = exponent + exponent_digits; + } + + position == end +} + +/// `StringTypeNarrower::isFloat()`. +fn is_float(value: &[u8]) -> bool { + is_numeric(value) && value.iter().any(|byte| matches!(byte, b'.' | b'e' | b'E')) +} + +/// `is_numeric($value) && (string) (int) $value === $value`: the canonical decimal form of an int64. +fn is_integer(value: &[u8]) -> bool { + std::str::from_utf8(value) + .ok() + .and_then(|string| string.parse::().ok()) + .is_some_and(|parsed| parsed.to_string().as_bytes() == value) +} + +/// `StringTemporalParts::hasExplicitDay()`. +fn has_explicit_day(value: &[u8]) -> bool { + let groups = value + .iter() + .enumerate() + .filter(|(index, byte)| byte.is_ascii_digit() && (*index == 0 || !value[index - 1].is_ascii_digit())) + .count(); + + if groups >= 3 { + return true; + } + + if groups == 1 && value.len() == 8 && value.iter().all(u8::is_ascii_digit) { + return true; + } + + groups >= 2 && value.windows(3).any(|window| window.iter().all(u8::is_ascii_alphabetic)) +} + +/// `StringTypeNarrower::isTimeZone()`'s `/^[+-]\d{2}:\d{2}$/` on a trimmed value. +fn is_offset(value: &[u8]) -> bool { + matches!(value, [b'+' | b'-', h1, h2, b':', m1, m2] if [h1, h2, m1, m2].iter().all(|byte| byte.is_ascii_digit())) +} + +/// `checkdate()`. +fn checkdate(month: i64, day: i64, year: i64) -> bool { + if !(1..=32767).contains(&year) || !(1..=12).contains(&month) || day < 1 { + return false; + } + + let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + let days = match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + + day <= days +} + +struct Column { + name: Vec, + kind: Kind, +} + +/// `ColumnTypes::observe()` over whole rows: names seeded `null` (then widened), names met only in rows start at +/// their first observed type; a `string` / `?string` column skips non-null cells. +pub struct Fold { + pub narrower: Narrower, + columns: Vec, + index: HashMap, usize>, + rows: u64, +} + +impl Fold { + pub fn new(names: Vec>, candidates: &[Vec]) -> Result { + let mut fold = Self { + narrower: Narrower::new(Candidates::from_codes(candidates)?)?, + columns: Vec::new(), + index: HashMap::new(), + rows: 0, + }; + + for name in names { + if !fold.index.contains_key(&name) { + fold.index.insert(name.clone(), fold.columns.len()); + fold.columns.push(Column { + name, + kind: Kind::of(Leaf::Null), + }); + } + } + + Ok(fold) + } + + /// One cell of the current row; `None` is a null cell. `position` caches the cell's column across rows. + pub fn observe(&mut self, name: &[u8], position: &mut Option, value: Option<&[u8]>) -> Result<(), PhpException> { + if position.is_none() { + *position = self.index.get(name).copied(); + } + + if value.is_some() && position.is_some_and(|position| self.columns[position].kind.is_string()) { + return Ok(()); + } + + let observed = Kind::of(match value { + None => Leaf::Null, + Some(value) if php_trim(value).len() != value.len() => Leaf::String, + Some(value) => match self.narrower.narrow(value)? { + Leaf::Null => Leaf::String, + leaf => leaf, + }, + }); + + match *position { + Some(position) => self.columns[position].kind = widen(self.columns[position].kind, observed), + None => { + self.index.insert(name.to_vec(), self.columns.len()); + *position = Some(self.columns.len()); + self.columns.push(Column { + name: name.to_vec(), + kind: observed, + }); + } + } + + Ok(()) + } + + pub fn end_row(&mut self) { + self.rows += 1; + } + + pub fn rows(&self) -> u64 { + self.rows + } + + /// Column name => type code, first-seen order. + pub fn types(&self) -> impl Iterator { + self.columns.iter().map(|column| (column.name.as_slice(), column.kind.code())) + } +} + +pub fn string_values(list: &ZendHashTable) -> Vec> { + list.values() + .filter_map(|zv| zv.zend_str().map(|value| value.as_bytes().to_vec())) + .collect() +} diff --git a/src/extension/flow-php-ext/src/csv/mod.rs b/src/extension/flow-php-ext/src/csv/mod.rs new file mode 100644 index 0000000000..979be674b8 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/mod.rs @@ -0,0 +1,263 @@ +//! Native CSV reading: records and fields from `tokenizer`, rows shaped exactly like +//! `CSVEncoder::decode()` + `CSVRowNormalizer::normalize()` shape them. + +pub mod fold; +pub mod tokenizer; + +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; + +use crate::ctx::{array_key_index, ht_insert_key, null_zval, write_slot, zval_str, HtKey}; +use crate::exception::ext_exception; +use crate::hydrate::RowValuesClass; +use tokenizer::{Dialect, Field, Record, Tokenizer}; + +/// A header name resolved once into the array key `array_combine()` would use for it. +enum HeaderKey { + Index(i64), + Str(ZBox), +} + +struct Header { + name: Vec, + key: HeaderKey, +} + +impl Header { + fn new(name: Vec) -> Self { + let key = match array_key_index(&name) { + Some(index) => HeaderKey::Index(index), + None => HeaderKey::Str(ZendStr::new(&name, false)), + }; + + Self { name, key } + } + + fn key(&self) -> HtKey<'_> { + match &self.key { + HeaderKey::Index(index) => HtKey::Index(*index), + HeaderKey::Str(name) => HtKey::Str(name), + } + } +} + +/// One key of `RawRowValues::$values`: `array_combine()` keeps a duplicated header at its first position with the +/// value of its last one. +struct Cell { + header_index: usize, + field_index: usize, +} + +pub struct CsvReader { + tokenizer: Tokenizer, + with_header: bool, + empty_to_null: bool, + headers: Option>, + cells: Vec, + record: Record, + first_row_pending: bool, +} + +impl CsvReader { + pub fn new( + separator: &[u8], + enclosure: &[u8], + escape: &[u8], + with_header: bool, + empty_to_null: bool, + remove_bom: bool, + ) -> Result { + let [separator] = separator else { + return Err(ext_exception("flow_php CSV separator must be exactly one byte")); + }; + let [enclosure] = enclosure else { + return Err(ext_exception("flow_php CSV enclosure must be exactly one byte")); + }; + let escape = match escape { + [] => None, + [escape] => Some(*escape), + _ => return Err(ext_exception("flow_php CSV escape must be empty or exactly one byte")), + }; + + Ok(Self { + tokenizer: Tokenizer::new( + Dialect { + separator: *separator, + enclosure: *enclosure, + escape, + }, + remove_bom, + ), + with_header, + empty_to_null, + headers: None, + cells: Vec::new(), + record: Record::default(), + first_row_pending: false, + }) + } + + pub fn feed(&mut self, chunk: &[u8]) { + self.tokenizer.feed(chunk); + } + + pub fn finish(&mut self) { + self.tokenizer.finish(); + } + + /// The resolved header, `[]` until the first record is available. + pub fn headers(&mut self) -> Vec<&[u8]> { + self.resolve_headers(); + + self.headers + .as_ref() + .map(|headers| headers.iter().map(|header| header.name.as_slice()).collect()) + .unwrap_or_default() + } + + /// Up to `batch_size` `RawRowValues`; an empty list when no complete record is buffered. + pub fn next(&mut self, batch_size: usize, class: &RowValuesClass) -> Result, PhpException> { + self.resolve_headers(); + + let mut batch = ZendHashTable::new(); + + let Some(headers) = self.headers.as_ref() else { + return Ok(batch); + }; + + while batch.len() < batch_size { + if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record) { + break; + } + + batch + .push(row_values(headers, &self.record, self.empty_to_null, class)) + .map_err(|e| ext_exception(format!("flow_php failed to collect CSV row values: {e:?}")))?; + } + + Ok(batch) + } + + /// Folds up to `limit` rows (all when `None`) into `fold`; returns how many it folded - fewer only when no + /// further complete record is buffered. + pub fn fold(&mut self, fold: &mut fold::Fold, limit: Option) -> Result { + self.resolve_headers(); + + let Some(headers) = self.headers.as_ref() else { + return Ok(0); + }; + + let mut folded = 0; + let mut positions = vec![None; self.cells.len()]; + + while limit.is_none_or(|limit| folded < limit) { + if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record) { + break; + } + + for (cell, position) in self.cells.iter().zip(positions.iter_mut()) { + fold.observe( + &headers[cell.header_index].name, + position, + cell_value(&self.record, cell.field_index, self.empty_to_null), + )?; + } + + fold.end_row(); + folded += 1; + } + + Ok(folded) + } + + /// `CSVEncoder::decode()`'s first line: the header when `withHeader`, else `e00, e01, ...` sized by + /// the first record, which then stays pending as the first row. + fn resolve_headers(&mut self) { + if self.headers.is_some() || !self.tokenizer.next(&mut self.record) { + return; + } + + let record = &self.record; + + self.headers = Some(if self.with_header { + (0..record.len()) + .map(|index| { + let trimmed = php_trim(match record.field(index) { + Field::Value(value) => value, + Field::Null | Field::Missing => b"", + }); + + Header::new(if trimmed.is_empty() { + format!("e{index:02}").into_bytes() + } else { + trimmed.to_vec() + }) + }) + .collect() + } else { + self.first_row_pending = true; + + (0..record.len()).map(|index| Header::new(format!("e{index:02}").into_bytes())).collect() + }); + + let headers = self.headers.as_ref().expect("resolved above"); + + for (index, header) in headers.iter().enumerate() { + match self.cells.iter_mut().find(|cell| headers[cell.header_index].name == header.name) { + Some(cell) => cell.field_index = index, + None => self.cells.push(Cell { + header_index: index, + field_index: index, + }), + } + } + } +} + +/// The first row is pending when auto headers were sized by it; otherwise the next complete record. +fn next_row(first_row_pending: &mut bool, tokenizer: &mut Tokenizer, record: &mut Record) -> bool { + std::mem::take(first_row_pending) || tokenizer.next(record) +} + +/// `CSVRowNormalizer::normalize()` for one cell: padding past the record's end, `emptyToNull`; `None` is null. +fn cell_value(record: &Record, index: usize, empty_to_null: bool) -> Option<&[u8]> { + match record.field(index) { + Field::Value(value) if !(empty_to_null && value.is_empty()) => Some(value), + Field::Value(_) | Field::Null => None, + Field::Missing if empty_to_null => None, + Field::Missing => Some(b""), + } +} + +/// `new RawRowValues(array_combine($headers, $normalizer->normalize($fields, count($headers))))`. +fn row_values(headers: &[Header], record: &Record, empty_to_null: bool, class: &RowValuesClass) -> ZBox { + let mut values = ZendHashTable::with_capacity(headers.len() as u32); + + for (index, header) in headers.iter().enumerate() { + let value = cell_value(record, index, empty_to_null).map_or_else(null_zval, zval_str); + + ht_insert_key(&mut values, &header.key(), value); + } + + let mut row_values = ZendObject::new(class.ce); + + let mut values_zv = Zval::new(); + values_zv.set_hashtable(values); + write_slot(&mut row_values, class.values_slot, values_zv); + + let mut metadata_zv = Zval::new(); + metadata_zv.set_hashtable(ZendHashTable::new()); + write_slot(&mut row_values, class.metadata_slot, metadata_zv); + + row_values +} + +/// PHP's `trim()`: `" \t\n\r\0\x0B"`, nothing else. +pub fn php_trim(bytes: &[u8]) -> &[u8] { + let is_trimmed = |byte: &u8| matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\0' | 0x0B); + let start = bytes.iter().position(|byte| !is_trimmed(byte)).unwrap_or(bytes.len()); + let end = bytes.iter().rposition(|byte| !is_trimmed(byte)).map_or(start, |end| end + 1); + + &bytes[start..end] +} diff --git a/src/extension/flow-php-ext/src/csv/tokenizer.rs b/src/extension/flow-php-ext/src/csv/tokenizer.rs new file mode 100644 index 0000000000..a3af8a56f0 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/tokenizer.rs @@ -0,0 +1,344 @@ +//! CSV records and fields exactly as the PHP path produces them: `CSVLineReader` assembles a record +//! (a `\n` ends it only outside an enclosure, the record is `rtrim($buffer, "\r\n")`, the first one +//! loses its BOM) and `str_getcsv()` splits it - a port of `php_fgetcsv()` (`ext/standard/file.c`). + +use std::ops::Range; + +use memchr::{memchr, memchr2}; + +pub struct Dialect { + pub separator: u8, + pub enclosure: u8, + pub escape: Option, +} + +/// One record's fields as ranges of `bytes`; no field at all is a blank line (`str_getcsv('')` is `[null]`). +#[derive(Default)] +pub struct Record { + bytes: Vec, + fields: Vec>, +} + +pub enum Field<'a> { + Value(&'a [u8]), + Null, + Missing, +} + +impl Record { + pub fn len(&self) -> usize { + self.fields.len().max(1) + } + + pub fn field(&self, index: usize) -> Field<'_> { + match self.fields.get(index) { + Some(range) => Field::Value(&self.bytes[range.clone()]), + None if index == 0 && self.fields.is_empty() => Field::Null, + None => Field::Missing, + } + } +} + +/// Where the record scan stands after the last byte it looked at. +#[derive(Clone, Copy)] +enum Scan { + FieldStart, + Unenclosed, + Enclosed, + Escaped, + ClosingOrDoubled, +} + +const BOMS: [&[u8]; 5] = [ + b"\xEF\xBB\xBF", + b"\xFF\xFE\x00\x00", + b"\x00\x00\xFE\xFF", + b"\xFF\xFE", + b"\xFE\xFF", +]; + +/// Resumable across chunk boundaries: `feed` appends bytes, `next` fills the next complete record or returns +/// `false` while it is not yet closed. A record left open at EOF is emitted as-is (PHP's `unterminated` case). +pub struct Tokenizer { + dialect: Dialect, + remove_bom: bool, + buffer: Vec, + record_start: usize, + scanned: usize, + scan: Scan, + finished: bool, + first_record: bool, +} + +impl Tokenizer { + pub fn new(dialect: Dialect, remove_bom: bool) -> Self { + Self { + dialect, + remove_bom, + buffer: Vec::new(), + record_start: 0, + scanned: 0, + scan: Scan::FieldStart, + finished: false, + first_record: true, + } + } + + pub fn feed(&mut self, chunk: &[u8]) { + if self.record_start > 0 { + self.buffer.drain(..self.record_start); + self.scanned -= self.record_start; + self.record_start = 0; + } + + self.buffer.extend_from_slice(chunk); + } + + pub fn finish(&mut self) { + self.finished = true; + } + + pub fn next(&mut self, record: &mut Record) -> bool { + let Some(range) = self.next_record() else { + return false; + }; + + self.split(range, record); + + true + } + + /// The next complete record as a range of `buffer`, after `rtrim("\r\n")` and the BOM strip. + fn next_record(&mut self) -> Option> { + let mut record = match self.scan_to_record_end() { + Some(newline) => { + let range = self.record_start..newline; + self.record_start = newline + 1; + self.scanned = self.record_start; + self.scan = Scan::FieldStart; + + range + } + None if self.finished && self.record_start < self.buffer.len() => { + let range = self.record_start..self.buffer.len(); + self.record_start = self.buffer.len(); + self.scanned = self.record_start; + self.scan = Scan::FieldStart; + + range + } + None => return None, + }; + + while record.end > record.start && matches!(self.buffer[record.end - 1], b'\r' | b'\n') { + record.end -= 1; + } + + if self.remove_bom && self.first_record { + if let Some(bom) = BOMS.iter().find(|bom| self.buffer[record.clone()].starts_with(bom)) { + record.start += bom.len(); + } + } + + self.first_record = false; + + Some(record) + } + + /// Advances the scan over unseen bytes; returns the position of the `\n` that ends the record. + fn scan_to_record_end(&mut self) -> Option { + let Dialect { separator, enclosure, escape } = self.dialect; + let bytes = &self.buffer; + let mut position = self.scanned; + let mut scan = self.scan; + + let found = loop { + if position >= bytes.len() { + break None; + } + + match scan { + Scan::FieldStart => { + let byte = bytes[position]; + + if byte == b'\n' { + break Some(position); + } + + scan = if byte == separator { + Scan::FieldStart + } else if byte == enclosure { + Scan::Enclosed + } else if is_space(byte) { + Scan::FieldStart + } else { + Scan::Unenclosed + }; + position += 1; + } + Scan::Unenclosed => match memchr2(separator, b'\n', &bytes[position..]) { + Some(offset) if bytes[position + offset] == b'\n' => break Some(position + offset), + Some(offset) => { + position += offset + 1; + scan = Scan::FieldStart; + } + None => position = bytes.len(), + }, + Scan::Enclosed => { + let special = match escape { + Some(escape) if escape != enclosure => memchr2(enclosure, escape, &bytes[position..]), + _ => memchr(enclosure, &bytes[position..]), + }; + + match special { + Some(offset) => { + scan = if bytes[position + offset] == enclosure { + Scan::ClosingOrDoubled + } else { + Scan::Escaped + }; + position += offset + 1; + } + None => position = bytes.len(), + } + } + Scan::Escaped => { + scan = Scan::Enclosed; + position += 1; + } + Scan::ClosingOrDoubled => { + if bytes[position] == enclosure { + scan = Scan::Enclosed; + position += 1; + } else { + scan = Scan::Unenclosed; + } + } + } + }; + + self.scanned = position; + self.scan = scan; + + found + } + + /// `str_getcsv($record, ...)` into `record`. + fn split(&self, range: Range, record: &mut Record) { + let Dialect { separator, enclosure, escape } = self.dialect; + let escape = escape.filter(|escape| *escape != enclosure); + let line = &self.buffer[range]; + // next_record() already stripped every trailing \r and \n, so php_fgetcsv()'s line end is always empty here + let limit = line.len(); + + record.bytes.clear(); + record.fields.clear(); + + if limit == 0 { + return; + } + + let mut position = 0; + + loop { + let field_start = record.bytes.len(); + + if position < limit { + let mut skipped = position; + + while skipped < limit && line[skipped] != separator && is_space(line[skipped]) { + skipped += 1; + } + + if skipped < limit && line[skipped] == enclosure { + position = skipped; + } + } + + let delimiter = if position < limit && line[position] == enclosure { + position += 1; + let mut hunk = position; + + let unterminated = loop { + let Some(offset) = (match escape { + Some(escape) => memchr2(enclosure, escape, &line[position..limit]), + None => memchr(enclosure, &line[position..limit]), + }) else { + break true; + }; + + position += offset; + + if line[position] != enclosure { + position += 2; + + if position > limit { + break true; + } + + continue; + } + + if position + 1 < limit && line[position + 1] == enclosure { + record.bytes.extend_from_slice(&line[hunk..=position]); + position += 2; + hunk = position; + + continue; + } + + record.bytes.extend_from_slice(&line[hunk..position]); + position += 1; + hunk = position; + + break false; + }; + + if unterminated { + record.bytes.extend_from_slice(&line[hunk..limit]); + hunk = limit; + position = limit; + } + + let delimiter = memchr(separator, &line[position..limit]).map(|offset| position + offset); + let end = delimiter.unwrap_or(limit); + + if hunk < end { + record.bytes.extend_from_slice(&line[hunk..end]); + } + + delimiter + } else { + let delimiter = memchr(separator, &line[position..limit]).map(|offset| position + offset); + + record.bytes.extend_from_slice(&line[position..delimiter.unwrap_or(limit)]); + + let trimmed = field_start + trailing_line_end(&record.bytes[field_start..]); + record.bytes.truncate(trimmed); + + delimiter + }; + + record.fields.push(field_start..record.bytes.len()); + + match delimiter { + Some(delimiter) => position = delimiter + 1, + None => break, + } + } + } +} + +/// `isspace()` in the C locale. +pub fn is_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0B | 0x0C | b'\r') +} + +/// `php_fgetcsv_lookup_trailing_spaces()`: the length without one trailing `\r\n`, `\n` or `\r`. +fn trailing_line_end(bytes: &[u8]) -> usize { + match bytes { + [.., b'\r', b'\n'] => bytes.len() - 2, + [.., b'\n'] | [.., b'\r'] => bytes.len() - 1, + _ => bytes.len(), + } +} diff --git a/src/extension/flow-php-ext/src/ctx.rs b/src/extension/flow-php-ext/src/ctx.rs index fe3b3dc8b6..be66349852 100644 --- a/src/extension/flow-php-ext/src/ctx.rs +++ b/src/extension/flow-php-ext/src/ctx.rs @@ -1,7 +1,7 @@ //! PHP-engine plumbing: class-entry/slot lookups, hashtable helpers, calls and //! per-instance caches (timezones, enums, callables). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use ext_php_rs::boxed::ZBox; use ext_php_rs::convert::IntoZvalDyn; @@ -487,6 +487,9 @@ pub struct Ctx { fn_json_encode: Option, fn_json_decode: Option, fn_json_validate: Option, + fn_date_parse: Option, + timezone_identifiers: Option>>, + timezone_construct: Option<&'static Function>, metadata_from_array: Option<&'static Function>, metadata_map_slot: Option, typed_row_values_slots: Option<(u32, u32)>, @@ -531,6 +534,9 @@ impl Ctx { fn_json_encode: None, fn_json_decode: None, fn_json_validate: None, + fn_date_parse: None, + timezone_identifiers: None, + timezone_construct: None, metadata_from_array: None, metadata_map_slot: None, typed_row_values_slots: None, @@ -900,6 +906,58 @@ impl Ctx { Ok(self.fn_json_validate.as_ref().expect("just initialized")) } + pub fn date_parse(&mut self) -> Result<&Function, PhpException> { + if self.fn_date_parse.is_none() { + self.fn_date_parse = Some(function_handle("date_parse")?); + } + + Ok(self.fn_date_parse.as_ref().expect("just initialized")) + } + + /// `DateTimeZone::listIdentifiers()`, fetched once per `Ctx`. + pub fn timezone_identifiers(&mut self) -> Result<&HashSet>, PhpException> { + if self.timezone_identifiers.is_none() { + let list = call_handle( + method_handle_ref("DateTimeZone", "listIdentifiers")?, + None, + &mut [], + "list timezone identifiers", + )?; + let identifiers = list + .array() + .ok_or_else(|| ext_exception("flow_php expected DateTimeZone::listIdentifiers() to return an array"))? + .values() + .filter_map(|zv| zv.zend_str().map(|name| name.as_bytes().to_vec())) + .collect(); + + self.timezone_identifiers = Some(identifiers); + } + + Ok(self.timezone_identifiers.as_ref().expect("just initialized")) + } + + /// Whether `new DateTimeZone($name)` succeeds; the exception it throws otherwise is discarded. + pub fn timezone_accepts(&mut self, name: &[u8]) -> Result { + if self.timezone_ce.is_none() { + self.timezone_ce = Some(find_class("DateTimeZone")?); + } + + let ce = self.timezone_ce.expect("just initialized"); + + if self.timezone_construct.is_none() { + self.timezone_construct = Some(ce_method_ref(ce, "__construct")?); + } + + let timezone = ZendObject::new(ce); + + Ok(call_handle_catching( + self.timezone_construct.expect("just initialized"), + Some(&timezone), + &mut [zval_str(name)], + ) + .is_ok()) + } + /// Static `Flow\ETL\Schema\Metadata::fromArray` handle - builds a Metadata /// value object from a decoded per-value metadata map (the rare path). pub fn metadata_from_array(&mut self) -> Result<&'static Function, PhpException> { diff --git a/src/extension/flow-php-ext/src/hydrate.rs b/src/extension/flow-php-ext/src/hydrate.rs index 67139fbfff..d7ae3caf21 100644 --- a/src/extension/flow-php-ext/src/hydrate.rs +++ b/src/extension/flow-php-ext/src/hydrate.rs @@ -347,6 +347,8 @@ pub struct AssemblyClasses { /// `Rows::conformed()` - the shape-only door `HydratedBatch` returns through, /// so both hydrators assemble the batch by the same rule pub rows_conformed: &'static Function, + /// `Rows::trusted()` - taken instead of `rows_conformed` when conform() would return every row unchanged + pub rows_trusted: &'static Function, pub schema_mismatch_ce: &'static ClassEntry, /// the base `PhpRowHydrator`'s guard catches, so a refusal is recognised on /// both paths by the same rule and anything else stays the caller's exception @@ -359,6 +361,7 @@ impl AssemblyClasses { Ok(Self { row_ce: find_class("Flow\\ETL\\Row")?, rows_conformed: ce_method_ref(find_class("Flow\\ETL\\Rows")?, "conformed")?, + rows_trusted: ce_method_ref(find_class("Flow\\ETL\\Rows")?, "trusted")?, schema_mismatch_ce: find_class("Flow\\ETL\\Exception\\SchemaMismatchException")?, types_exception_ce: find_class("Flow\\Types\\Exception\\Exception")?, value_does_not_match: ce_method_ref( diff --git a/src/extension/flow-php-ext/src/json_check.rs b/src/extension/flow-php-ext/src/json_check.rs new file mode 100644 index 0000000000..fa2ebec9c7 --- /dev/null +++ b/src/extension/flow-php-ext/src/json_check.rs @@ -0,0 +1,21 @@ +/// Necessary condition for nesting deeper than json_validate's 511, not a validator: such a cell holds at least +/// 512 `[`/`{` bytes. Brackets inside strings only cost an extra PHP call. +pub fn json_nesting_within_php_depth(bytes: &[u8]) -> bool { + memchr::memchr2_iter(b'[', b'{', bytes).take(512).count() <= 511 +} + +/// Necessary condition for an unpaired surrogate escape, not a validator: it needs a `\u` followed by `d8`-`df`. +/// Any such escape, paired or not, is left to PHP. +pub fn json_without_surrogate_escape(bytes: &[u8]) -> bool { + !memchr::memchr_iter(b'\\', bytes).any(|at| { + matches!(bytes.get(at + 1..at + 4), Some([b'u', b'd' | b'D', b'8'..=b'9' | b'a'..=b'f' | b'A'..=b'F'])) + }) +} + +/// `true` is json_validate()'s verdict; `false` is never authoritative - the caller asks PHP. +pub fn json_valid(bytes: &[u8]) -> bool { + std::str::from_utf8(bytes).is_ok() + && serde_json::from_slice::(bytes).is_ok() + && json_nesting_within_php_depth(bytes) + && json_without_surrogate_escape(bytes) +} diff --git a/src/extension/flow-php-ext/src/lib.rs b/src/extension/flow-php-ext/src/lib.rs index 522c6459eb..a22baa5a84 100644 --- a/src/extension/flow-php-ext/src/lib.rs +++ b/src/extension/flow-php-ext/src/lib.rs @@ -1,9 +1,11 @@ mod cast; +mod csv; mod ctx; mod encode; mod exception; mod format; mod hydrate; +mod json_check; mod plan; mod values; @@ -15,7 +17,7 @@ use ext_php_rs::types::{ZendHashTable, Zval}; use ext_php_rs::zend::ModuleEntry; use ext_php_rs::{info_table_end, info_table_row, info_table_start}; -use crate::ctx::{zval_str, Ctx}; +use crate::ctx::{ht_insert, zval_str, Ctx}; use crate::encode::{build_encode_plan, encode_typed_row, expect_object, ht_for_each, read_slot, EncodePlan}; use crate::exception::ext_exception; use crate::format::Reader; @@ -247,6 +249,120 @@ impl RustRowHydratorNative { } } +/// Native counterpart of `CSVLineReader` + `CSVEncoder::decode()`: bytes in, `RawRowValues` out, +/// byte-identical to the PHP path. Resumable - `feed` any chunk size, `finish` at EOF. +#[php_class] +#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustCSVReaderNative")] +pub struct RustCSVReaderNative { + reader: csv::CsvReader, + row_values_class: hydrate::RowValuesClass, +} + +#[php_impl] +impl RustCSVReaderNative { + pub fn __construct( + separator: BinarySlice, + enclosure: BinarySlice, + escape: BinarySlice, + with_header: bool, + empty_to_null: bool, + remove_bom: bool, + ) -> PhpResult { + Ok(Self { + reader: csv::CsvReader::new(&separator, &enclosure, &escape, with_header, empty_to_null, remove_bom)?, + row_values_class: hydrate::RowValuesClass::resolve()?, + }) + } + + pub fn feed(&mut self, chunk: BinarySlice) { + self.reader.feed(&chunk); + } + + pub fn finish(&mut self) { + self.reader.finish(); + } + + pub fn headers(&mut self) -> PhpResult { + let headers = self.reader.headers(); + let mut list = ZendHashTable::with_capacity(headers.len() as u32); + + for header in headers { + list.push(zval_str(header)) + .map_err(|e| ext_exception(format!("flow_php failed to collect a CSV header: {e:?}")))?; + } + + let mut zv = Zval::new(); + zv.set_hashtable(list); + + Ok(zv) + } + + /// Folds up to `limit` buffered rows (-1: all) into `fold`; returns how many it folded. + pub fn fold(&mut self, fold: &mut RustColumnFoldNative, limit: i64) -> PhpResult { + let limit = match limit { + -1 => None, + limit => Some( + u64::try_from(limit).map_err(|_| ext_exception("flow_php CSV fold limit must be -1 or at least 0"))?, + ), + }; + + Ok(self.reader.fold(&mut fold.fold, limit)? as i64) + } + + pub fn next(&mut self, batch_size: i64) -> PhpResult { + let batch_size = usize::try_from(batch_size) + .ok() + .filter(|size| *size > 0) + .ok_or_else(|| ext_exception("flow_php CSV batch size must be greater than 0"))?; + + let mut zv = Zval::new(); + zv.set_hashtable(self.reader.next(batch_size, &self.row_values_class)?); + + Ok(zv) + } +} + +/// `ColumnTypes::observe()` over native CSV rows: a column fold seeded with the header names and gated by the +/// candidate types' `toString()` codes. HTML and XML candidates are never passed - the PHP side folds those itself. +#[php_class] +#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustColumnFoldNative")] +pub struct RustColumnFoldNative { + fold: csv::fold::Fold, +} + +#[php_impl] +impl RustColumnFoldNative { + pub fn __construct(names: &ZendHashTable, candidates: &ZendHashTable) -> PhpResult { + Ok(Self { + fold: csv::fold::Fold::new(csv::fold::string_values(names), &csv::fold::string_values(candidates))?, + }) + } + + /// `StringTypeNarrower::narrow($value)->toString()` for the fold's candidates. + #[php(name = "narrowOne")] + pub fn narrow_one(&mut self, value: BinarySlice) -> PhpResult { + Ok(self.fold.narrower.narrow(&value)?.code().to_string()) + } + + /// Column name => type `toString()`, first-seen order. + pub fn types(&self) -> Zval { + let mut types = ZendHashTable::new(); + + for (name, code) in self.fold.types() { + ht_insert(&mut types, name, zval_str(code.as_bytes())); + } + + let mut zv = Zval::new(); + zv.set_hashtable(types); + + zv + } + + pub fn rows(&self) -> i64 { + self.fold.rows() as i64 + } +} + #[php_module] #[php(startup = "module_startup")] pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { @@ -255,4 +371,6 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { .info_function(php_module_info) .class::() .class::() + .class::() + .class::() } diff --git a/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt index 12662fcb4b..a466225608 100644 --- a/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt @@ -134,6 +134,32 @@ $datasets = [ ], ], 'empty' => [schema(int_schema('id')), []], + // every column present in every row: the batch skips Row::conform(), which would return each row unchanged + 'all_present' => [ + schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('a'), float_schema('p', nullable: true)), + [ + new RawRowValues(['id' => '1', 'name' => 'x', 'a' => 'yes', 'p' => '1.5']), + new RawRowValues(['id' => '2', 'name' => null, 'a' => 'no', 'p' => null]), + new RawRowValues(['p' => 3, 'a' => true, 'name' => 'z', 'id' => 3]), + ], + ], + 'absent_nullable' => [ + schema(int_schema('id'), str_schema('name', nullable: true)), + [ + new RawRowValues(['id' => 1, 'name' => 'a']), + new RawRowValues(['id' => 2, 'name' => 'b']), + new RawRowValues(['id' => 3, 'name' => 'c']), + new RawRowValues(['id' => 4]), + new RawRowValues(['id' => 5, 'name' => 'e']), + ], + ], + 'numeric_names' => [ + schema(int_schema('id'), str_schema('7'), int_schema('10', nullable: true)), + [ + new RawRowValues(['id' => '1', '7' => 'seven', '10' => '10']), + new RawRowValues(['10' => null, '7' => 'eight', 'id' => 2]), + ], + ], ]; $php = new PhpRowHydrator(); @@ -166,6 +192,124 @@ printf( serialize($php->hydrate($batch, $mutated)) === serialize($native->hydrate($batch, $mutated)) ? 'yes' : 'NO', ); +// row 3 lacks a NOT NULL column and row 5 is refused by its cast: the cast refusal is raised while the batch is +// built, before Rows::conformed() could report the absence, so both paths name row 5 +$refused = schema(int_schema('id'), int_schema('n')); +$refusedBatch = [ + new RawRowValues(['id' => 0, 'n' => 0]), + new RawRowValues(['id' => 1, 'n' => 1]), + new RawRowValues(['id' => 2, 'n' => 2]), + new RawRowValues(['id' => 3]), + new RawRowValues(['id' => 4, 'n' => 4]), + new RawRowValues(['id' => 5, 'n' => 'five']), +]; +$refusals = []; + +foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { + try { + $hydrator->hydrate($refusedBatch, $refused); + $refusals[$side] = 'none'; + } catch (Throwable $e) { + $refusals[$side] = $e::class . ': ' . $e->getMessage(); + } +} + +printf("%-16s identical:%s\n", 'absent_then_refused', $refusals['php'] === $refusals['native'] ? 'yes' : 'NO'); +echo $refusals['native'], "\n"; + +// a value held by reference is cast by value: the hydrated row must not change when the caller's variable does +$referenced = [ + 'ref_json' => [schema(json_schema('c')), '{"a":1}', static fn(&$v): array => ['c' => &$v]], + 'ref_uuid' => [schema(uuid_schema('c')), '01234567-89ab-4def-8123-456789abcdef', static fn(&$v): array => ['c' => &$v]], + 'ref_null' => [schema(str_schema('c', nullable: true)), null, static fn(&$v): array => ['c' => &$v]], + 'ref_datetime' => [schema(datetime_schema('c')), new DateTimeImmutable('2020-01-01 00:00:00'), static fn(&$v): array => ['c' => &$v]], + 'ref_list_item' => [schema(list_schema('c', type_list(type_positive_integer()))), 5, static fn(&$v): array => ['c' => [&$v]]], + 'ref_st_element' => [ + schema(structure_schema('c', type_structure(['s' => type_non_empty_string()]))), + 'text', + static fn(&$v): array => ['c' => ['s' => &$v]], + ], +]; + +foreach ($referenced as $label => [$s, $value, $wrap]) { + $hydrated = []; + + foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { + $v = $value; + $rows = $hydrator->hydrate([new RawRowValues($wrap($v))], $s); + $v = 'mutated'; + $hydrated[$side] = serialize($rows); + } + + printf("%-16s hydrate:%s\n", $label, $hydrated['php'] === $hydrated['native'] ? 'yes' : 'NO'); +} + +$isoStrings = [ + '2026-07-13T10:20:30+00:00', + "2026-07-13T10:20:30+00:00\n", + "2026-07-13T10:20:30+00:00\n\n", + '2026-07-13T10:20:30Z', + '2026-07-13T10:20:30+05', + '2026-07-13T10:20:30+0530', + '2026-07-13T10:20:30+05:30', + '2026-07-13T10:20:30-12:00', + '2026-07-13 10:20:30', + '2026-07-13T10:20', + '2026-07-13T10:20:30.1Z', + '2026-07-13T10:20:30.12Z', + '2026-07-13T10:20:30.123Z', + '2026-07-13T10:20:30.1234Z', + '2026-07-13T10:20:30.12345Z', + '2026-07-13T10:20:30.123456Z', + '2026-07-13T10:20:30.1234567Z', + '2026-07-13T10:20:30.12345678Z', + '2026-07-13T10:20:30.123456789Z', + '2026-07-13T10:20:30.1234567890Z', + '2026-02-30T00:00:00Z', + '2024-02-29T00:00:00Z', + '2023-02-29T00:00:00Z', + '0000-01-01T00:00:00Z', + '2026-07-13T25:99:99Z', + '２０２６-07-13T10:20:30Z', + 'now', +]; + +// the gate's only outcome-changing check is checkdate(): PHP throws on an impossible day, the constructor rolls it over +foreach (['0000', '0001', '1900', '2000', '2023', '2024', '2100'] as $year) { + for ($month = 0; $month <= 13; $month++) { + for ($day = 0; $day <= 32; $day++) { + $isoStrings[] = sprintf('%s-%02d-%02dT00:00:00Z', $year, $month, $day); + } + } +} + +$isoSchema = schema(datetime_schema('at')); + +foreach (['UTC', 'Europe/Warsaw'] as $zone) { + ini_set('date.timezone', $zone); + $mismatches = 0; + + foreach ($isoStrings as $isoString) { + $results = []; + + foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { + try { + $at = $hydrator->hydrate([new RawRowValues(['at' => $isoString])], $isoSchema)->first()->get('at'); + $results[$side] = $at::class . ' ' . $at->format('Y-m-d\TH:i:s.uP') . ' ' . $at->getTimezone()->getName(); + } catch (Throwable $e) { + $results[$side] = $e::class . ': ' . $e->getMessage(); + } + } + + if ($results['php'] !== $results['native']) { + $mismatches++; + echo ' ', var_export($isoString, true), ": php[{$results['php']}] native[{$results['native']}]\n"; + } + } + + printf("iso datetime %-13s %d strings, %d mismatches\n", $zone, count($isoStrings), $mismatches); +} + printf("native class registered:%s\n", class_exists(RustRowHydratorNative::class, false) ? 'yes' : 'NO'); ?> --EXPECT-- @@ -179,6 +323,19 @@ fill_and_metadata hydrate:yes all_optional_st hydrate:yes interleaved_st hydrate:yes empty hydrate:yes +all_present hydrate:yes +absent_nullable hydrate:yes +numeric_names hydrate:yes list_promotion php:[33.0,65.5] native:[33.0,65.5] schema_mutation hydrate:yes +absent_then_refused identical:yes +Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "n" (row 5): could not convert 'five' (string) to integer +ref_json hydrate:yes +ref_uuid hydrate:yes +ref_null hydrate:yes +ref_datetime hydrate:yes +ref_list_item hydrate:yes +ref_st_element hydrate:yes +iso datetime UTC 3261 strings, 0 mismatches +iso datetime Europe/Warsaw 3261 strings, 0 mismatches native class registered:yes diff --git a/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt index faec92d3ba..0d54410bdc 100644 --- a/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt +++ b/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt @@ -85,10 +85,20 @@ $throwingBatch = [ // an absent NOT-NULL column aborts before any row_values insertion - a different leak path $missingBatch = [new RawRowValues(['name' => 'no id here'])]; -$cycle = static function () use ($batch, $throwingBatch, $missingBatch, $schema): void { +$jsonSchema = schema(json_schema('json')); + +$cycle = static function () use ($batch, $throwingBatch, $missingBatch, $schema, $jsonSchema): void { $native = new RustRowHydratorNative(); $native->hydrate($batch, $schema); + foreach (json_leak_cells() as $cell) { + try { + $native->hydrate([new RawRowValues(['json' => $cell])], $jsonSchema); + } catch (Throwable) { + // a refused cell aborts its batch; nothing it touched may leak + } + } + try { $native->hydrate($throwingBatch, $schema); } catch (Throwable) { diff --git a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt new file mode 100644 index 0000000000..a12bdbdb8a --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt @@ -0,0 +1,44 @@ +--TEST-- +native CSV fields match str_getcsv() byte for byte on every tokenizer edge case +--SKIPIF-- + +--FILE-- + '612c622c63', + 'quoted' => '2261222c226222', + 'backslash_quote' => '22615c2262222c63', + 'doubled_quote' => '2261222262222c63', + 'space_before_enclosure' => '2261222c2020226222', + 'junk_after_close' => '612c226222782c63', + 'unterminated' => '22756e7465726d696e617465642c61', + 'blank' => '', + 'trailing_sep' => '612c622c', +]; + +foreach ($cases as $case => $hex) { + $input = hex2bin($hex); + $reader = new RustCSVReaderNative(',', '"', '\\', false, false, false); + $reader->feed($input . "\n"); + $reader->finish(); + + $fields = array_values($reader->next(10)[0]->values); + $expected = str_getcsv($input, ',', '"', '\\'); + + echo $case, ': ', $fields === $expected ? 'identical' : 'FAIL ' . json_encode(array_map(static fn(?string $f): ?string => $f === null ? null : bin2hex($f), $fields)), "\n"; +} +?> +--EXPECT-- +plain: identical +quoted: identical +backslash_quote: identical +doubled_quote: identical +space_before_enclosure: identical +junk_after_close: identical +unterminated: identical +blank: identical +trailing_sep: identical diff --git a/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt b/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt new file mode 100644 index 0000000000..1e5e8655f8 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt @@ -0,0 +1,73 @@ +--TEST-- +native CSV reading matches the PHP path for every dialect and read option +--SKIPIF-- + +--FILE-- + +--EXPECT-- +";" ' "\\" 111: identical +";" ' "\\" 011: identical +";" ' "\\" 101: identical +";" ' "\\" 110: identical +";" ' "" 111: identical +";" ' "" 011: identical +";" ' "" 101: identical +";" ' "" 110: identical +";" " "\\" 111: identical +";" " "\\" 011: identical +";" " "\\" 101: identical +";" " "\\" 110: identical +";" " "" 111: identical +";" " "" 011: identical +";" " "" 101: identical +";" " "" 110: identical +"|" ' "\\" 111: identical +"|" ' "\\" 011: identical +"|" ' "\\" 101: identical +"|" ' "\\" 110: identical +"|" ' "" 111: identical +"|" ' "" 011: identical +"|" ' "" 101: identical +"|" ' "" 110: identical +"|" " "\\" 111: identical +"|" " "\\" 011: identical +"|" " "\\" 101: identical +"|" " "\\" 110: identical +"|" " "" 111: identical +"|" " "" 011: identical +"|" " "" 101: identical +"|" " "" 110: identical +"\t" ' "\\" 111: identical +"\t" ' "\\" 011: identical +"\t" ' "\\" 101: identical +"\t" ' "\\" 110: identical +"\t" ' "" 111: identical +"\t" ' "" 011: identical +"\t" ' "" 101: identical +"\t" ' "" 110: identical +"\t" " "\\" 111: identical +"\t" " "\\" 011: identical +"\t" " "\\" 101: identical +"\t" " "\\" 110: identical +"\t" " "" 111: identical +"\t" " "" 011: identical +"\t" " "" 101: identical +"\t" " "" 110: identical diff --git a/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt b/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt new file mode 100644 index 0000000000..d1ac835ed3 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt @@ -0,0 +1,38 @@ +--TEST-- +native CSV records span lines exactly like CSVLineReader: quoted LF and CRLF, open at EOF, blank lines, BOM before a quote +--SKIPIF-- + +--FILE-- + "a,b\n\"x\ny\",1\n2,3\n", + 'quoted CRLF' => "a,b\r\n\"x\r\ny\",1\r\n2,3\r\n", + 'escaped quote then LF' => "a,b\n\"x\\\"\ny\",1\n", + 'doubled quote then LF' => "a,b\n\"x\"\"\ny\",1\n", + 'bare quote in unenclosed field' => "a,b\nx\"y,1\n\"p\",2\n", + 'open at EOF' => "a,b\n1,\"never closed\nstill open\n", + 'open at EOF without LF' => "a,b\n1,\"never closed", + 'blank lines' => "a,b\n\n1,2\n\n\n", + 'extra CRs' => "a,b\r\r\n1,2\r\r\r\n", + 'BOM then quote' => "\xEF\xBB\xBF\"a\",b\n1,2\n", + 'BOM then open quote' => "\xEF\xBB\xBF\"a\nb\",c\n1,2\n", +]; + +foreach ($cases as $case => $raw) { + assert_csv_identical($case, csv_php_rows($raw, ',', '"', '\\'), csv_native_rows($raw, ',', '"', '\\')); +} +?> +--EXPECT-- +quoted LF: identical +quoted CRLF: identical +escaped quote then LF: identical +doubled quote then LF: identical +bare quote in unenclosed field: identical +open at EOF: identical +open at EOF without LF: identical +blank lines: identical +extra CRs: identical +BOM then quote: identical +BOM then open quote: identical diff --git a/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt b/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt new file mode 100644 index 0000000000..9064f10c3f --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt @@ -0,0 +1,24 @@ +--TEST-- +native CSV rows do not depend on where chunk boundaries fall +--SKIPIF-- + +--FILE-- + +--EXPECT-- +whole input vs PHP path: identical +chunk 1: identical +chunk 2: identical +chunk 3: identical +chunk 7: identical +chunk 4096: identical diff --git a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt new file mode 100644 index 0000000000..c447a601d0 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt @@ -0,0 +1,50 @@ +--TEST-- +native CSV rows pad and truncate to the header like CSVRowNormalizer, under both emptyToNull settings +--SKIPIF-- + +--FILE-- + +--EXPECT-- +emptyToNull=1 withHeader=1: identical +emptyToNull=1 withHeader=0: identical +emptyToNull=0 withHeader=1: identical +emptyToNull=0 withHeader=0: identical +array(5) { + [0]=> + string(2) "id" + [1]=> + string(4) "name" + [2]=> + string(3) "e02" + [3]=> + string(1) "5" + [4]=> + string(2) "id" +} +array(4) { + ["id"]=> + NULL + ["name"]=> + NULL + ["e02"]=> + NULL + [5]=> + NULL +} diff --git a/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt new file mode 100644 index 0000000000..b2d946369e --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt @@ -0,0 +1,52 @@ +--TEST-- +repeated native CSV reading does not leak memory +--SKIPIF-- + +--FILE-- +feed($chunk); + $reader->next(2); + } + + $reader->finish(); + $reader->headers(); + + while ($reader->next(2) !== []) { + } + + $unfinished = new RustCSVReaderNative(';', "'", '', false, false, false); + $unfinished->feed("a;'open\n"); + $unfinished->next(10); + + try { + new RustCSVReaderNative(',,', '"', '\\', true, true, true); + throw new LogicException('an invalid separator was accepted'); + } catch (Flow\Floe\Exception\ExtensionException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 10000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt new file mode 100644 index 0000000000..33c1460008 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt @@ -0,0 +1,21 @@ +--TEST-- +native narrowing matches StringTypeNarrower::narrow() on its fixtures, the byte-level traps and a fuzz corpus +--SKIPIF-- + +--FILE-- += 80); +var_dump(count($corpus) >= 300); +assert_narrow_parity('default candidates', InferredTypes::default()->toArray(), $corpus); +?> +--EXPECT-- +bool(true) +bool(true) +default candidates: identical diff --git a/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt b/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt new file mode 100644 index 0000000000..b34c9502e7 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt @@ -0,0 +1,42 @@ +--TEST-- +native narrowing skips exactly the rungs a restricted candidate set leaves out +--SKIPIF-- + +--FILE-- + new InferredTypes(type_string()), + 'integer' => new InferredTypes(type_integer()), + 'float integer' => new InferredTypes(type_float(), type_integer()), + 'date' => new InferredTypes(type_date()), + 'datetime' => new InferredTypes(type_datetime()), + 'boolean timezone' => new InferredTypes(type_boolean(), type_time_zone()), + 'json uuid' => new InferredTypes(type_json(), type_uuid()), +] as $label => $candidates) { + assert_narrow_parity($label, $candidates->toArray(), $corpus); +} +?> +--EXPECT-- +all strings: identical +integer: identical +float integer: identical +date: identical +datetime: identical +boolean timezone: identical +json uuid: identical diff --git a/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt new file mode 100644 index 0000000000..972de86d42 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt @@ -0,0 +1,71 @@ +--TEST-- +the native fold widens every pair of CSV leaf types exactly like TypeWidener +--SKIPIF-- + +--FILE-- + [null, type_null()], + 'string' => ['x', type_string()], + 'json' => ['{"a":1}', type_json()], + 'uuid' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479', type_uuid()], + 'float' => ['1.5', type_float()], + 'integer' => ['1', type_integer()], + 'datetime' => ['2024-01-01 10:00:00', type_datetime()], + 'date' => ['2024-01-01', type_date()], + 'boolean' => ['true', type_boolean()], + 'timezone' => ['UTC', type_time_zone()], +]; +$candidates = array_map(static fn($type): string => $type->toString(), InferredTypes::default()->toArray()); +$widener = new TypeWidener(); +$pairs = 0; +$mismatches = 0; + +foreach ($leaves as $left => [$leftCell, $leftType]) { + foreach ($leaves as $right => [$rightCell, $rightType]) { + $csv = fopen('php://memory', 'rb+'); + fputcsv($csv, ['c'], ',', '"', '\\'); + fputcsv($csv, [$leftCell ?? ''], ',', '"', '\\'); + fputcsv($csv, [$rightCell ?? ''], ',', '"', '\\'); + rewind($csv); + + $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); + $reader->feed((string) stream_get_contents($csv)); + $reader->finish(); + $fold = new RustColumnFoldNative([], $candidates); + $reader->fold($fold, -1); + + $expected = $widener->widen($leftType, $rightType)->toString(); + $actual = $fold->types()['c']; + $pairs++; + + if ($expected !== $actual) { + $mismatches++; + echo "{$left} + {$right}: TypeWidener={$expected} native={$actual}\n"; + } + } +} + +echo "{$pairs} pairs, {$mismatches} mismatches\n"; +?> +--EXPECT-- +100 pairs, 0 mismatches diff --git a/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt new file mode 100644 index 0000000000..7f7c292b4d --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt @@ -0,0 +1,47 @@ +--TEST-- +repeated native CSV folding - including every PHP callback and a rejected time zone offset - does not leak memory +--SKIPIF-- + +--FILE-- + $type->toString(), InferredTypes::default()->toArray()); +$raw = "id,json,at,zone,offset,dup,dup\n1,\"{\"\"a\"\":1}\",2024-01-01 10:00:00,Europe/Warsaw,+99:60,x,\n2,[1],2024-02-30,UTC,+02:00,,y\n"; + +$cycle = static function () use ($candidates, $raw): void { + $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); + $reader->feed($raw); + $reader->finish(); + $fold = new RustColumnFoldNative(['id', 'json'], $candidates); + $reader->fold($fold, -1); + $fold->types(); + $fold->rows(); + $fold->narrowOne('+99:60'); + $fold->narrowOne('{"a":[1,2]}'); + $fold->narrowOne('2024-01-01'); + + foreach (json_leak_cells() as $cell) { + $fold->narrowOne($cell); + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 10000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt b/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt new file mode 100644 index 0000000000..6ccff95294 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt @@ -0,0 +1,54 @@ +--TEST-- +native CSV rows match the PHP path over a seeded fuzz corpus, every dialect and chunk size +--SKIPIF-- + +--FILE-- + [',', "a, \"b\nc\",d\ne\n"], + 'whitespace separator before an enclosure' => ["\t", "h1\th2\th3\na\t\t\"b\"\n"], + 'trailing carriage return in an unenclosed field' => [',', "h1,h2\na\r,b\n"], +]; + +foreach ($named as $case => [$separator, $raw]) { + assert_csv_identical( + $case, + csv_php_rows($raw, $separator, '"', '\\', false), + csv_native_rows($raw, $separator, '"', '\\', false), + ); +} + +mt_srand(50); +$alphabet = ['a', 'b', ' ', "\t", ',', ';', '"', "'", '\\', "\n", "\r"]; +$dialects = [[',', '"', '\\'], [';', '"', '\\'], [',', "'", '\\'], [',', '"', ''], ["\t", '"', '\\']]; +$mismatches = 0; + +for ($i = 0; $i < 2000; $i++) { + $raw = ''; + + for ($j = mt_rand(0, 40); $j > 0; $j--) { + $raw .= $alphabet[mt_rand(0, count($alphabet) - 1)]; + } + + [$separator, $enclosure, $escape] = $dialects[mt_rand(0, count($dialects) - 1)]; + $withHeader = mt_rand(0, 1) === 1; + $emptyToNull = mt_rand(0, 1) === 1; + $expected = csv_php_rows($raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull); + + foreach ([1, 7, 4096] as $chunk) { + if (csv_native_rows($raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull, true, $chunk) !== $expected) { + $mismatches++; + echo 'MISMATCH ', json_encode([$raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull, $chunk]), "\n"; + } + } +} + +echo "fuzz mismatches: {$mismatches}\n"; +?> +--EXPECT-- +blanks before a multi-line enclosure: identical +whitespace separator before an enclosure: identical +trailing carriage return in an unenclosed field: identical +fuzz mismatches: 0 diff --git a/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt new file mode 100644 index 0000000000..9f97f6eb6e --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt @@ -0,0 +1,30 @@ +--TEST-- +native CSV classes reject an invalid dialect, batch size, fold limit and HTML/XML candidates +--SKIPIF-- + +--FILE-- + new RustCSVReaderNative(',,', '"', '\\', true, true, true)); +expect_exception(static fn() => new RustCSVReaderNative(',', '', '\\', true, true, true)); +expect_exception(static fn() => new RustCSVReaderNative(',', '"', '\\\\', true, true, true)); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(0)); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(-1)); +$fold = new RustColumnFoldNative([], ['integer']); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->fold($fold, -2)); +expect_exception(static fn() => new RustColumnFoldNative([], ['integer', 'html'])); +expect_exception(static fn() => new RustColumnFoldNative([], ['xml'])); +?> +--EXPECT-- +Flow\Floe\Exception\ExtensionException: flow_php CSV separator must be exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV enclosure must be exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV escape must be empty or exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 +Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 +Flow\Floe\Exception\ExtensionException: flow_php CSV fold limit must be -1 or at least 0 +Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively +Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively diff --git a/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt new file mode 100644 index 0000000000..ae839d5e4d --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt @@ -0,0 +1,71 @@ +--TEST-- +native JSON cast matches PhpRowHydrator on the JSON parity cases and a deterministic fuzz corpus +--SKIPIF-- + +--FILE-- + substr($fuzzed, 0, $at) . $insert . substr($fuzzed, $at), + 1 => substr($fuzzed, 0, $at) . substr($fuzzed, $at + 1), + default => substr($fuzzed, 0, $at) . $insert . substr($fuzzed, $at + 1), + }; + } + } + + $cases["fuzz_{$i}"] = $fuzzed; +} + +$schema = schema(json_schema('j', nullable: false)); +$php = new PhpRowHydrator(); +$native = new NativeRowHydrator(); + +$outcome = static function (Flow\ETL\Row\Hydrator $hydrator, string $value) use ($schema): string { + try { + return 'json ' . $hydrator->hydrate([new RawRowValues(['j' => $value])], $schema)->first()->get('j')->toString(); + } catch (Throwable $e) { + return $e::class . ': ' . $e->getMessage(); + } +}; + +$mismatches = 0; + +foreach ($cases as $name => $value) { + if ($outcome($php, $value) !== $outcome($native, $value)) { + $mismatches++; + echo ' ', $name, ' ', bin2hex(substr($value, 0, 60)), "\n"; + } +} + +printf("json cast parity: %d cases, %d mismatches\n", count($cases), $mismatches); +?> +--EXPECT-- +json cast parity: 20054 cases, 0 mismatches diff --git a/src/extension/flow-php-ext/tests/phpt/053_csv_fold_json_parity.phpt b/src/extension/flow-php-ext/tests/phpt/053_csv_fold_json_parity.phpt new file mode 100644 index 0000000000..3b822c71b2 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/053_csv_fold_json_parity.phpt @@ -0,0 +1,14 @@ +--TEST-- +native CSV narrowing matches StringTypeNarrower::narrow() on the JSON parity cases +--SKIPIF-- + +--FILE-- +toArray(), array_values(json_parity_cases())); +?> +--EXPECT-- +json cases: identical diff --git a/src/extension/flow-php-ext/tests/phpt/bootstrap.php b/src/extension/flow-php-ext/tests/phpt/bootstrap.php index 0913d9121c..59fa05318c 100644 --- a/src/extension/flow-php-ext/tests/phpt/bootstrap.php +++ b/src/extension/flow-php-ext/tests/phpt/bootstrap.php @@ -175,3 +175,392 @@ function expect_exception(callable $fn): void echo get_class($e), ': ', $e->getMessage(), "\n"; } } + +/** + * The PHP CSV path - CSVLineReader + CSVEncoder::decode() - as `[headers, list of RawRowValues::$values]`. + * + * @return array{list, list>} + */ +function csv_php_rows( + string $raw, + string $separator, + string $enclosure, + string $escape, + bool $withHeader = true, + bool $emptyToNull = true, + bool $removeBOM = true, +): array { + $encoder = new Flow\ETL\Adapter\CSV\CSVEncoder( + withHeader: $withHeader, + separator: $separator, + enclosure: $enclosure, + escape: $escape, + emptyToNull: $emptyToNull, + ); + $lines = new Flow\ETL\Adapter\CSV\CSVLineReader($enclosure, $separator, $escape, removeBOM: $removeBOM); + $rows = []; + + foreach ($lines->readLines( + new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.csv'), $raw), + ) as $line) { + foreach ($encoder->decode([$line]) as $values) { + $rows[] = $values->values; + } + } + + return [$encoder->headers() ?? [], $rows]; +} + +/** + * RustCSVReaderNative fed `$raw` in `$chunk`-byte pieces, as `[headers, list of RawRowValues::$values]`. + * + * @param positive-int $chunk + * + * @return array{list, list>} + */ +function csv_native_rows( + string $raw, + string $separator, + string $enclosure, + string $escape, + bool $withHeader = true, + bool $emptyToNull = true, + bool $removeBOM = true, + int $chunk = 4096, +): array { + $reader = new Flow\ETL\Adapter\CSV\RustCSVReaderNative( + $separator, + $enclosure, + $escape, + $withHeader, + $emptyToNull, + $removeBOM, + ); + $rows = []; + $drain = static function () use ($reader, &$rows): void { + while (($batch = $reader->next(3)) !== []) { + foreach ($batch as $values) { + $rows[] = $values->values; + } + } + }; + + foreach ($raw === '' ? [] : str_split($raw, $chunk) as $piece) { + $reader->feed($piece); + $drain(); + } + + $reader->finish(); + $drain(); + + return [$reader->headers(), $rows]; +} + +/** + * Prints `