From bb6929ed5d3e761510d776be32e6b966a16ddf8d Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Mon, 21 Sep 2026 11:35:46 +0200 Subject: [PATCH 1/4] perf(flow-php/etl): cut CSV read overhead - skip type narrowing once a column saturates to string - return rows that already match the schema from conform - drop the per-field coercion closure in CSV decode - hoist the stream handle out of the readLines/iterate loops --- .../src/Flow/ETL/Adapter/CSV/CSVEncoder.php | 6 +- .../Adapter/CSV/Tests/Unit/CSVEncoderTest.php | 24 +++++ src/core/etl/src/Flow/ETL/Row.php | 15 +++ .../Flow/ETL/Schema/Inference/ColumnTypes.php | 19 +++- .../etl/tests/Flow/ETL/Tests/Unit/RowTest.php | 35 ++++++ .../Unit/Schema/Inference/ColumnTypesTest.php | 58 ++++++++++ .../Schema/Inference/SchemaInferrerTest.php | 48 +++++++++ .../Stream/NativeLocalSourceStream.php | 26 +++-- .../NativeLocalSourceStreamTest.php | 48 +++++++++ .../Native/String/StringTypeNarrowerTest.php | 100 ++++++++++++++++++ .../Types/Tests/Unit/Type/TypeWidenerTest.php | 85 +++++++++++++++ 11 files changed, 451 insertions(+), 13 deletions(-) diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php index 1bbfb691d2..8dc8f6d79b 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEncoder.php @@ -89,10 +89,8 @@ public function decode(array $batch): array $maps = []; foreach ($batch as $line) { - $fields = array_values(array_map( - static fn(mixed $field): ?string => is_string($field) ? $field : null, - str_getcsv($line, $this->separator, $this->enclosure, $this->escape), - )); + /** @var list $fields */ + $fields = str_getcsv($line, $this->separator, $this->enclosure, $this->escape); if ($this->headers === null) { if ($this->withHeader) { diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php index 750e0f27ad..36d7a16fa2 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEncoderTest.php @@ -74,6 +74,30 @@ public function test_decode_turns_empty_fields_into_null_by_default(): void static::assertSame(['id' => '1', 'name' => null], (new CSVEncoder())->decode(['id,name', '1,'])[0]->values); } + public function test_decoding_a_blank_line_yields_a_single_null_field(): void + { + static::assertSame( + ['e00' => null], + (new CSVEncoder(withHeader: false, emptyToNull: false))->decode([''])[0]->values, + ); + } + + public function test_decoding_a_line_with_a_custom_separator_enclosure_and_escape_yields_strings(): void + { + static::assertSame( + ['id' => '1', 'name' => 'a;b'], + (new CSVEncoder(separator: ';', enclosure: "'", escape: '|'))->decode(['id;name', "1;'a;b'"])[0]->values, + ); + } + + public function test_decoding_a_line_with_empty_quoted_fields_yields_empty_strings(): void + { + static::assertSame( + ['id' => '', 'name' => ''], + (new CSVEncoder(emptyToNull: false))->decode(['id,name', '"",""'])[0]->values, + ); + } + public function test_encode_renders_array_values_as_json(): void { static::assertSame( diff --git a/src/core/etl/src/Flow/ETL/Row.php b/src/core/etl/src/Flow/ETL/Row.php index 3c3f1b6bea..8977093468 100644 --- a/src/core/etl/src/Flow/ETL/Row.php +++ b/src/core/etl/src/Flow/ETL/Row.php @@ -165,6 +165,21 @@ public function values(): array private function conform(Schema $schema, bool $checkValues): self { $definitions = $schema->definitions(); + + if (count($this->values) === count($definitions) && array_keys($this->values) === array_keys($definitions)) { + foreach ($definitions as $name => $definition) { + if ( + $checkValues + ? !$definition->matches($this->values[$name]) + : $this->values[$name] === null && !$definition->isNullable() + ) { + throw ColumnMismatchException::valueDoesNotMatch($definition, $this->values[$name]); + } + } + + return $this; + } + $matched = []; $taken = 0; diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php index a1a8154de3..adecc32882 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php @@ -7,7 +7,9 @@ use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; use Flow\Types\Type; +use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Native\NullType; +use Flow\Types\Type\Native\StringType; use Flow\Types\Type\TypeNarrower; use Flow\Types\Type\TypeWidener; @@ -78,6 +80,19 @@ public function observe(RawRowValues $row): void { /** @var mixed $value */ foreach ($row->values as $name => $value) { + $current = array_key_exists($name, $this->types) ? $this->types[$name] : null; + + if ( + $value !== null + && ( + $current instanceof StringType + || $current instanceof OptionalType + && $current->base() instanceof StringType + ) + ) { + continue; + } + $observed = match (true) { $value === null => type_null(), is_string($value) && trim($value) !== $value => type_string(), @@ -88,9 +103,7 @@ public function observe(RawRowValues $row): void $observed = type_string(); } - $this->types[$name] = array_key_exists($name, $this->types) - ? $this->widener->widen($this->types[$name], $observed) - : $observed; + $this->types[$name] = $current === null ? $observed : $this->widener->widen($current, $observed); } $this->rows++; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php index 5407bca89d..f93ee1d54c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php @@ -26,6 +26,31 @@ final class RowTest extends FlowTestCase { + public function test_conform_to_a_schema_in_a_different_order_reorders_the_values(): void + { + static::assertSame( + ['b' => 2, 'a' => 1], + row(['a' => 1, 'b' => 2])->conformTo(schema(int_schema('b'), int_schema('a')))->values(), + ); + } + + public function test_conform_to_a_schema_the_row_already_matches_returns_an_equal_row(): void + { + $row = row(['a' => 1, 'b' => 'x']); + + static::assertEquals($row, $row->conformTo(schema(int_schema('a'), str_schema('b')))); + } + + public function test_conform_to_a_schema_with_as_many_but_other_columns_refuses_the_row(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'Row does not match its schema: column "b" declared by the schema is missing from the row', + ); + + row(['a' => 1, 'z' => 2])->conformTo(schema(int_schema('a'), int_schema('b'))); + } + public function test_conform_to_does_not_validate_a_non_null_value(): void { // the caller produced the value by casting to the column's type - conformTo() checks the shape only @@ -154,6 +179,16 @@ public function test_match_to_reports_no_row_coordinate_of_its_own(): void row(['a' => 1])->matchTo(schema()); } + public function test_match_to_with_the_same_keys_still_validates_every_value(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'Row does not match its schema: column "c": could not convert 3 (integer) to string', + ); + + row(['a' => 1, 'b' => 'x', 'c' => 3])->matchTo(schema(int_schema('a'), str_schema('b'), str_schema('c'))); + } + public function test_get_throws_when_the_column_is_absent(): void { $this->expectException(InvalidArgumentException::class); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php index fbcd5480a7..52c598934b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php @@ -150,6 +150,10 @@ public static function widenedColumns(): array 'integer then text' => [['1', 'x'], type_string()], 'date then datetime' => [['2024-01-01', '2024-01-01 10:00'], type_datetime()], 'both booleans' => [['true', 'false'], type_boolean()], + 'free text then datetime' => [['free text', '2024-01-01 10:00:00'], type_string()], + 'free text then integer' => [['free text', '42'], type_string()], + 'free text then json' => [['free text', '{"a":1}'], type_string()], + 'free text then uuid' => [['free text', 'f47ac10b-58cc-4372-a567-0e02b2c3d479'], type_string()], ]; } @@ -174,6 +178,19 @@ public function test_a_name_met_only_in_a_row_is_appended_after_the_header_names static::assertSame(['a', 'b', 'c'], array_keys($columns->schema(ColumnTypesMother::floor())->definitions())); } + public function test_a_null_first_row_then_free_text_yields_optional_string(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => 'free text'])); + $columns->observe(new RawRowValues(['c' => '42'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_a_numeric_key_becomes_a_string_column_name(): void { $columns = ColumnTypesMother::fromStrings(); @@ -184,6 +201,19 @@ public function test_a_numeric_key_becomes_a_string_column_name(): void static::assertSame('1', $columns->schema(ColumnTypesMother::floor())->get('1')->entry()->name()); } + public function test_a_saturated_string_column_widens_to_optional_string_on_a_later_null(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => 'free text'])); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => 'more text'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_a_source_with_no_names_and_no_rows_yields_an_empty_schema(): void { static::assertEquals(new Schema(), ColumnTypesMother::fromStrings()->schema(ColumnTypesMother::floor())); @@ -231,6 +261,19 @@ public function test_an_integer_column_is_declared_nullable(): void static::assertTrue($definition->isNullable()); } + public function test_an_optional_non_string_column_is_not_saturated(): void + { + $columns = ColumnTypesMother::fromStrings(); + $columns->observe(new RawRowValues(['c' => '1'])); + $columns->observe(new RawRowValues(['c' => null])); + $columns->observe(new RawRowValues(['c' => '1.5'])); + + static::assertEquals( + new Schema(definition_from_type('c', type_optional(type_float()), nullable: true)), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_merge_is_associative_over_three_partials(): void { $left = ColumnTypesMother::fromStrings(['a']); @@ -383,6 +426,21 @@ public function test_null_never_reaches_the_ladder(): void ); } + public function test_observing_a_column_name_absent_from_the_header_still_records_its_type(): void + { + $columns = ColumnTypesMother::fromStrings(['a']); + $columns->observe(new RawRowValues(['a' => 'free text', 'b' => '1'])); + $columns->observe(new RawRowValues(['a' => 'more text', 'b' => '2'])); + + static::assertEquals( + new Schema( + definition_from_type('a', type_string(), nullable: true), + definition_from_type('b', type_integer(), nullable: true), + ), + $columns->schema(ColumnTypesMother::floor()), + ); + } + public function test_padded_cells_are_read_as_text_because_cast_does_not_trim(): void { $padded = ColumnTypesMother::fromStrings(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php index 256e4ef765..63b6d88b3b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php @@ -19,6 +19,7 @@ use function array_keys; use function array_slice; use function Flow\ETL\DSL\definition_from_type; +use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; @@ -77,6 +78,33 @@ public static function threeSourcesOfTenRows(): array return $sources; } + public function test_a_column_that_is_free_text_in_the_first_source_and_numeric_in_the_second_widens_to_string(): void + { + $fixture = [ + [new RawRowValues(['id' => '1', 'v' => 'free text']), new RawRowValues(['id' => '2', 'v' => 'more text'])], + [new RawRowValues(['id' => '3', 'v' => '42']), new RawRowValues(['id' => '4', 'v' => '43'])], + ]; + $inferrer = new SchemaInferrer( + new SchemaInference(), + new StringTypeNarrower(InferredTypes::default()->toArray()), + ); + + static::assertEquals( + new Schema( + definition_from_type('id', type_integer(), nullable: true), + definition_from_type('v', type_string(), nullable: true), + ), + $inferrer->infer(['id', 'v'], (new RecordingSources($fixture))->sources()), + ); + static::assertEquals( + definition_from_type('v', type_integer(), nullable: true), + $inferrer + ->sniff(['id', 'v'], (new RecordingSources($fixture))->source(1), -1) + ->schema(ColumnTypesMother::floor()) + ->get('v'), + ); + } + public function test_a_row_less_source_does_not_spend_a_files_to_sniff_slot(): void { $sources = new RecordingSources([ @@ -319,6 +347,26 @@ public function test_the_candidate_set_reaches_the_floor_through_infer(): void ); } + public function test_union_by_name_over_sources_where_one_column_saturates_to_string(): void + { + $sources = new RecordingSources([ + [new RawRowValues(['id' => '1', 'v' => 'free text'])], + [new RawRowValues(['id' => '2', 'v' => '42', 'w' => '2024-01-01'])], + ]); + + static::assertEquals( + new Schema( + definition_from_type('id', type_integer(), nullable: true), + definition_from_type('v', type_string(), nullable: true), + definition_from_type('w', type_date(), nullable: true), + ), + (new SchemaInferrer( + new SchemaInference(unionByName: true), + new StringTypeNarrower(InferredTypes::default()->toArray()), + ))->infer([], $sources->sources()), + ); + } + public function test_unbounded_bounds_read_every_row_of_every_source(): void { $sources = new RecordingSources(self::threeSourcesOfTenRows()); diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php index c02b413642..6926401185 100644 --- a/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php @@ -103,16 +103,23 @@ public function iterate(int $length = 1): Generator throw new RuntimeException('Cannot read from closed stream'); } - fseek($this->handle(), 0); + $handle = $this->handle(); + + fseek($handle, 0); - while (!feof($this->handle())) { - $string = fread($this->handle(), $length); + while (!feof($handle)) { + $string = fread($handle, $length); if ($string === false) { break; } yield $string; + + // close() can run while the consumer holds the generator, and feof() on the dead handle is a TypeError + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } } } @@ -145,16 +152,23 @@ public function readLines(string $separator = "\n", ?int $length = null): Genera throw new RuntimeException('Cannot read from closed stream'); } - fseek($this->handle(), 0); + $handle = $this->handle(); + + fseek($handle, 0); - while (!feof($this->handle())) { - $line = stream_get_line($this->handle(), PHP_INT_MAX, $separator); + while (!feof($handle)) { + $line = stream_get_line($handle, PHP_INT_MAX, $separator); if ($line === false) { break; } yield $line; + + // close() can run while the consumer holds the generator, and feof() on the dead handle is a TypeError + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } } } diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php index ebe50d6c70..93ba9cf0cd 100644 --- a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php @@ -4,6 +4,7 @@ namespace Flow\Filesystem\Tests\Integration; +use Flow\Filesystem\Exception\RuntimeException; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Tests\Context\ReadLinesContext; use Generator; @@ -32,6 +33,40 @@ public static function line_lengths(): Generator yield [1024]; } + public function test_closing_a_stream_while_its_lines_are_being_iterated(): void + { + $this->givenFileExists(__DIR__ . '/var/file.txt', "x\ny\nz"); + + $stream = native_local_filesystem()->readFrom(path(__DIR__ . '/var/file.txt')); + $lines = $stream->readLines(); + + static::assertSame('x', $lines->current()); + + $stream->close(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Cannot read from closed stream'); + + $lines->next(); + } + + public function test_closing_a_stream_while_it_is_being_iterated(): void + { + $this->givenFileExists(__DIR__ . '/var/file.txt', 'xyz'); + + $stream = native_local_filesystem()->readFrom(path(__DIR__ . '/var/file.txt')); + $chunks = $stream->iterate(); + + static::assertSame('x', $chunks->current()); + + $stream->close(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Cannot read from closed stream'); + + $chunks->next(); + } + public function test_iterating_through_blob(): void { $content = <<<'TEXT' @@ -100,6 +135,19 @@ public function test_reading_lines_from_file(int $lineLength): void $stream->close(); } + public function test_read_lines_from_a_closed_stream_throws(): void + { + $this->givenFileExists(__DIR__ . '/var/file.txt', "x\ny"); + + $stream = native_local_filesystem()->readFrom(path(__DIR__ . '/var/file.txt')); + $stream->close(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Cannot read from closed stream'); + + $stream->readLines()->current(); + } + /** * @param non-empty-string $separator * @param null|int<1, max> $length diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/String/StringTypeNarrowerTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/String/StringTypeNarrowerTest.php index 2e9c435c64..c29a837949 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/String/StringTypeNarrowerTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/String/StringTypeNarrowerTest.php @@ -5,7 +5,9 @@ namespace Flow\Types\Tests\Unit\Type\Native\String; use DateTimeInterface; +use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Native\String\StringTypeNarrower; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; @@ -24,6 +26,98 @@ final class StringTypeNarrowerTest extends TestCase { + /** + * @return array + */ + public static function fixtureValues(): array + { + return [ + 'value true' => ['true'], + 'value false' => ['false'], + 'value yes' => ['yes'], + 'value no' => ['no'], + 'value on' => ['on'], + 'value off' => ['off'], + 'value 0' => ['0'], + 'value not bool' => ['not bool'], + 'value not date time' => ['not date time'], + 'value 2021-13-01' => ['2021-13-01'], + 'value now' => ['now'], + 'value midnight' => ['midnight'], + 'value today' => ['today'], + 'value yesterday' => ['yesterday'], + 'value tomorrow' => ['tomorrow'], + 'value +24h' => ['+24h'], + 'value 00:00:00' => ['00:00:00'], + 'value 2023-01-01 +10 hours' => ['2023-01-01 +10 hours'], + 'value Thursday, 02-Jun-2022 16:58:35 UTC' => ['Thursday, 02-Jun-2022 16:58:35 UTC'], + 'value 2022-06-02T16:58:35+0000' => ['2022-06-02T16:58:35+0000'], + 'value 2022-06-02T16:58:35+00:00' => ['2022-06-02T16:58:35+00:00'], + 'value Thu, 02 Jun 22 16:58:35 +0000' => ['Thu, 02 Jun 22 16:58:35 +0000'], + 'value Thursday, 02-Jun-22 16:58:35 UTC' => ['Thursday, 02-Jun-22 16:58:35 UTC'], + 'value Thu, 02 Jun 2022 16:58:35 +0000' => ['Thu, 02 Jun 2022 16:58:35 +0000'], + 'value 2024-01' => ['2024-01'], + 'value 12/31/2024' => ['12/31/2024'], + 'value 2024-01-01' => ['2024-01-01'], + 'value 2024-01-01 10:00' => ['2024-01-01 10:00'], + 'value 1.0' => ['1.0'], + 'value 2.1E-5' => ['2.1E-5'], + 'value 2.1e-5' => ['2.1e-5'], + 'value 0.0' => ['0.0'], + 'value not float' => ['not float'], + 'value 1' => ['1'], + 'value 1.0.0' => ['1.0.0'], + 'value 20240305' => ['20240305'], + 'value
1
' => [ + '
1
', + ], + 'value not html' => ['not html'], + 'value not integer' => ['not integer'], + 'value 112312312' => ['112312312'], + 'value 11_2312_312' => ['11_2312_312'], + 'value 20240101' => ['20240101'], + 'value 19991231' => ['19991231'], + 'value 1012024' => ['1012024'], + 'value {"foo":"bar"}' => ['{"foo":"bar"}'], + 'value [{"foo":"bar"}]' => ['[{"foo":"bar"}]'], + 'value not json' => ['not json'], + 'value null' => ['null'], + 'value NULL' => ['NULL'], + 'value Nil' => ['Nil'], + 'value nil' => ['nil'], + 'value not null' => ['not null'], + 'the empty string' => [''], + 'value UTC' => ['UTC'], + 'value America/New_York' => ['America/New_York'], + 'value Europe/London' => ['Europe/London'], + 'value Europe/Warsaw' => ['Europe/Warsaw'], + 'value Asia/Tokyo' => ['Asia/Tokyo'], + 'value Australia/Sydney' => ['Australia/Sydney'], + 'value +00:00' => ['+00:00'], + 'value +05:30' => ['+05:30'], + 'value -08:00' => ['-08:00'], + 'value A' => ['A'], + 'value B' => ['B'], + 'value Z' => ['Z'], + 'value PST' => ['PST'], + 'value EST' => ['EST'], + 'value CET' => ['CET'], + 'value not a timezone' => ['not a timezone'], + 'value Invalid/Timezone' => ['Invalid/Timezone'], + 'value 2023-01-01' => ['2023-01-01'], + 'value 123' => ['123'], + 'value f47ac10b-58cc-4372-a567-0e02b2c3d479' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479'], + 'value not uuid' => ['not uuid'], + 'value bar' => ['bar'], + 'value not xml' => ['not xml'], + 'value ['1' => ['1'], + 'value
x
' => ['
x
'], + 'value +02:00' => ['+02:00'], + 'value Europe/Nowhere' => ['Europe/Nowhere'], + ]; + } + public function test_detecting_boolean(): void { $narrower = new StringTypeNarrower(); @@ -231,6 +325,12 @@ public function test_a_rung_outside_the_emitted_types_is_skipped(): void static::assertEquals(type_xml(), (new StringTypeNarrower())->narrow('1')); } + #[DataProvider('fixtureValues')] + public function test_narrowing_never_returns_an_optional_type(string $value): void + { + static::assertNotInstanceOf(OptionalType::class, (new StringTypeNarrower())->narrow($value)); + } + public function test_time_zone_identifiers_are_matched_after_the_cache_is_warm(): void { foreach ([new StringTypeNarrower(), new StringTypeNarrower()] as $narrower) { diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/TypeWidenerTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/TypeWidenerTest.php index 4f7eda4ac5..8aec6a937a 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/TypeWidenerTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/TypeWidenerTest.php @@ -4,6 +4,7 @@ namespace Flow\Types\Tests\Unit\Type; +use Flow\Types\Tests\Unit\Type\Fixtures\SomeEnum; use Flow\Types\Type; use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\TypeWidener; @@ -12,22 +13,41 @@ use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use ReflectionMethod; +use stdClass; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_array; use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_callable; +use function Flow\Types\DSL\type_class_string; use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_datetime; use function Flow\Types\DSL\type_empty_array; +use function Flow\Types\DSL\type_enum; use function Flow\Types\DSL\type_float; +use function Flow\Types\DSL\type_html; +use function Flow\Types\DSL\type_html_element; +use function Flow\Types\DSL\type_instance_of; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_json; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_literal; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_non_empty_string; use function Flow\Types\DSL\type_null; +use function Flow\Types\DSL\type_numeric_string; +use function Flow\Types\DSL\type_object; use function Flow\Types\DSL\type_optional; +use function Flow\Types\DSL\type_positive_integer; +use function Flow\Types\DSL\type_resource; +use function Flow\Types\DSL\type_scalar; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; +use function Flow\Types\DSL\type_time; +use function Flow\Types\DSL\type_time_zone; +use function Flow\Types\DSL\type_uuid; +use function Flow\Types\DSL\type_xml; +use function Flow\Types\DSL\type_xml_element; final class TypeWidenerTest extends TestCase { @@ -212,6 +232,45 @@ public static function provideWidenCases(): Generator ]; } + /** + * Every concrete non-null, non-optional type the DSL builds - a superset of any inference candidate set. + * + * @return Generator}> + */ + public static function provideNonNullTypes(): Generator + { + yield 'array' => [type_array()]; + yield 'boolean' => [type_boolean()]; + yield 'callable' => [type_callable()]; + yield 'class string' => [type_class_string()]; + yield 'date' => [type_date()]; + yield 'datetime' => [type_datetime()]; + yield 'empty array' => [type_empty_array()]; + yield 'enum' => [type_enum(SomeEnum::class)]; + yield 'float' => [type_float()]; + yield 'html' => [type_html()]; + yield 'html element' => [type_html_element()]; + yield 'instance of' => [type_instance_of(stdClass::class)]; + yield 'integer' => [type_integer()]; + yield 'json' => [type_json()]; + yield 'list' => [type_list(type_integer())]; + yield 'literal' => [type_literal('x')]; + yield 'map' => [type_map(type_string(), type_integer())]; + yield 'non empty string' => [type_non_empty_string()]; + yield 'numeric string' => [type_numeric_string()]; + yield 'object' => [type_object()]; + yield 'positive integer' => [type_positive_integer()]; + yield 'resource' => [type_resource()]; + yield 'scalar' => [type_scalar()]; + yield 'string' => [type_string()]; + yield 'structure' => [type_structure(['a' => type_integer()])]; + yield 'time' => [type_time()]; + yield 'time zone' => [type_time_zone()]; + yield 'uuid' => [type_uuid()]; + yield 'xml' => [type_xml()]; + yield 'xml element' => [type_xml_element()]; + } + public static function provideStructureCases(): Generator { yield 'identical structures are unchanged' => [ @@ -341,6 +400,32 @@ public static function provideStructureCases(): Generator ]; } + /** + * @param Type $candidate + */ + #[DataProvider('provideNonNullTypes')] + public function test_widening_optional_string_with_any_non_null_candidate_type_yields_optional_string(Type $candidate): void + { + static::assertEquals( + type_optional(type_string()), + (new TypeWidener())->widen(type_optional(type_string()), $candidate), + ); + } + + /** + * @param Type $candidate + */ + #[DataProvider('provideNonNullTypes')] + public function test_widening_string_with_any_non_null_candidate_type_yields_string(Type $candidate): void + { + static::assertEquals(type_string(), (new TypeWidener())->widen(type_string(), $candidate)); + } + + public function test_widening_string_with_null_yields_optional_string(): void + { + static::assertEquals(type_optional(type_string()), (new TypeWidener())->widen(type_string(), type_null())); + } + /** * @param Type $left * @param Type $right From 62ce34b1629d1ed13d82d0b6d712fa52c199cf39 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Mon, 21 Sep 2026 14:53:51 +0200 Subject: [PATCH 2/4] perf(flow-php/flow-php-ext): read and infer CSV natively - Rust CSV tokenizer and schema fold behind the optional extension - adaptive CSVOpenSource: NativeCSVOpenSource / PhpCSVOpenSource - SniffsColumnTypes lets a sample fold itself in SchemaInferrer - escaped or bare enclosure no longer glues two records together - PCRE-limit fallback for record boundaries on huge records --- documentation/upgrading.md | 17 + .../Flow/ETL/Adapter/CSV/CSVEnclosureScan.php | 81 ++++ .../Flow/ETL/Adapter/CSV/CSVFileReader.php | 20 +- .../Flow/ETL/Adapter/CSV/CSVFileSample.php | 56 +++ .../Flow/ETL/Adapter/CSV/CSVLineReader.php | 24 +- .../Flow/ETL/Adapter/CSV/CSVOpenSource.php | 47 +- .../ETL/Adapter/CSV/CSVRecordBoundary.php | 56 +++ .../Flow/ETL/Adapter/CSV/CSVSourceOpener.php | 26 +- .../ETL/Adapter/CSV/NativeCSVOpenSource.php | 158 +++++++ .../Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php | 63 +++ .../CSV/Tests/Context/CSVFixtureContext.php | 243 ++++++++++ .../Context/CSVRecordBoundaryContext.php | 38 ++ .../Double/LengthCapturingFilesystem.php | 84 ++++ .../Double/LengthCapturingSourceStream.php | 7 + .../Tests/Fixtures/bare_quote_then_row.csv | 3 + .../Tests/Fixtures/doubled_quote_then_row.csv | 2 + .../Fixtures/escaped_quote_no_escape_char.csv | 3 + .../Tests/Fixtures/escaped_quote_then_row.csv | 3 + .../Fixtures/escaped_quote_with_newline.csv | 4 + .../Fixtures/escaped_quote_with_separator.csv | 3 + .../Adapter/CSV/Tests/Fixtures/fold_traps.csv | 4 + .../Tests/Integration/CSVExtractorTest.php | 143 ++++++ .../Tests/Integration/CSVFileReaderTest.php | 32 +- .../Tests/Integration/CSVFileSampleTest.php | 93 ++++ .../Tests/Integration/CSVLineReaderTest.php | 2 +- .../Tests/Integration/CSVSourceOpenerTest.php | 3 +- .../Integration/NativeCSVOpenSourceTest.php | 335 +++++++++++++ ...ourceTest.php => PhpCSVOpenSourceTest.php} | 16 +- .../CSV/Tests/Unit/CSVEnclosureScanTest.php | 27 ++ .../CSV/Tests/Unit/CSVLineReaderTest.php | 152 +++++- .../CSV/Tests/Unit/CSVRecordBoundaryTest.php | 34 ++ .../CSV/Tests/Unit/CSVSourceOpenerTest.php | 87 ++++ .../Flow/ETL/Schema/Inference/ColumnTypes.php | 18 + .../ETL/Schema/Inference/SchemaInferrer.php | 14 +- .../ETL/Schema/Inference/SchemaSampler.php | 5 +- .../Schema/Inference/SniffsColumnTypes.php | 28 ++ .../ETL/Tests/Double/SpySniffingSample.php | 51 ++ .../ETL/Tests/Mother/ColumnTypesMother.php | 11 + .../Unit/Schema/Inference/ColumnTypesTest.php | 45 ++ .../Schema/Inference/SchemaInferrerTest.php | 39 ++ src/extension/flow-php-ext/Cargo.lock | 1 + src/extension/flow-php-ext/Cargo.toml | 1 + src/extension/flow-php-ext/composer.json | 1 + .../ETL/Adapter/CSV/RustCSVReaderNative.php | 66 +++ .../ETL/Adapter/CSV/RustColumnFoldNative.php | 46 ++ src/extension/flow-php-ext/src/csv/fold.rs | 446 ++++++++++++++++++ src/extension/flow-php-ext/src/csv/mod.rs | 263 +++++++++++ .../flow-php-ext/src/csv/tokenizer.rs | 344 ++++++++++++++ src/extension/flow-php-ext/src/ctx.rs | 60 ++- src/extension/flow-php-ext/src/lib.rs | 119 ++++- .../tests/phpt/040_csv_tokenizer_parity.phpt | 44 ++ .../tests/phpt/041_csv_dialect_options.phpt | 73 +++ .../tests/phpt/042_csv_multiline_records.phpt | 38 ++ .../tests/phpt/043_csv_chunk_boundaries.phpt | 24 + .../tests/phpt/044_csv_ragged_rows.phpt | 50 ++ .../tests/phpt/045_csv_no_leaks.phpt | 52 ++ .../tests/phpt/046_csv_narrow_parity.phpt | 21 + .../phpt/047_csv_narrow_candidate_gating.phpt | 42 ++ .../tests/phpt/048_csv_widen_parity.phpt | 71 +++ .../tests/phpt/049_csv_fold_no_leaks.phpt | 43 ++ .../tests/phpt/050_csv_tokenizer_fuzz.phpt | 54 +++ .../tests/phpt/051_csv_guards.phpt | 30 ++ .../flow-php-ext/tests/phpt/bootstrap.php | 294 ++++++++++++ 63 files changed, 4141 insertions(+), 119 deletions(-) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{CSVOpenSourceTest.php => PhpCSVOpenSourceTest.php} (81%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php create mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php create mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php create mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php create mode 100644 src/extension/flow-php-ext/src/csv/fold.rs create mode 100644 src/extension/flow-php-ext/src/csv/mod.rs create mode 100644 src/extension/flow-php-ext/src/csv/tokenizer.rs create mode 100644 src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 75bc0a4d67..dca41dedcf 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -221,6 +221,23 @@ after it shift by one. `Stage::physical` is new - see the core documentation. | `new Explain\TreeLayout($details, declarations: true)` | `new Explain\Outline(declarations: true)`, the layout takes no arguments | | `new Explain\BoxLayout($details)` | `new Explain\BoxLayout()` | +### 22) `flow-php/etl-adapter-csv` - an escaped or bare enclosure no longer merges a record with the next line + +| Input (3 lines) | Before | After | +|------------------------------|----------------------------------------|---------------------------------------------------| +| `a,b` / `"x\"y",1` / `"p",2` | 1 row: `{"a":"x\\\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\\\"y","b":1}`, `{"a":"p","b":2}` | +| `a,b` / `x"y,1` / `"p",2` | 1 row: `{"a":"x\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\"y","b":1}`, `{"a":"p","b":2}` | + +### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVLineReader` takes the separator and escape + +| Before | After | +|----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `new CSVLineReader($enclosure, $charactersReadInLine, $removeBOM)` | `new CSVLineReader($enclosure, $separator, $escape, $charactersReadInLine, $removeBOM)` | +| `new CSVOpenSource($stream, $dialect, $encoder, $lineReader)` | `new PhpCSVOpenSource($stream, $encoder, $lineReader)`; `CSVOpenSource` is its interface | +| `$open->stream`, `$open->dialect`, `$open->encoder`, `$open->lineReader` | removed | +| `CSVFileReader::samples()` yields `Generator`s | yields `CSVFileSample` (`IteratorAggregate`); `$unit->getIterator()` for the generator | +| `CSVFileReader::sample($source)` | `new CSVFileSample($opener, $source)` | + --- ## Upgrading from 0.43.x to 0.44.x diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php new file mode 100644 index 0000000000..6b3ae0fb27 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVEnclosureScan.php @@ -0,0 +1,81 @@ +blanks = str_replace($separator, '', self::BLANKS); + $this->specials = $escape === $enclosure ? $enclosure : $enclosure . $escape; + } + + public function endsOutsideAnEnclosure(string $buffer): bool + { + $length = strlen($buffer); + $position = 0; + + while (true) { + $fieldStart = $position + strspn($buffer, $this->blanks, $position); + + if ($fieldStart < $length && $buffer[$fieldStart] === $this->enclosure) { + $position = $fieldStart + 1; + + while (true) { + $position += strcspn($buffer, $this->specials, $position); + + if ($position >= $length) { + return false; + } + + if ($buffer[$position] !== $this->enclosure) { + $position += 2; + + continue; + } + + if (($position + 1) < $length && $buffer[$position + 1] === $this->enclosure) { + $position += 2; + + continue; + } + + $position++; + + break; + } + } + + $nextSeparator = strpos($buffer, $this->separator, $position); + + if ($nextSeparator === false) { + return true; + } + + $position = $nextSeparator + 1; + } + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php index e937af1719..2d90dc60a4 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php @@ -79,28 +79,14 @@ public function header(): CSVHeader } /** - * @return Generator - */ - public function sample(SourceFile $source): Generator - { - $open = $this->opener->open($source); - - try { - yield from $open->records(); - } finally { - $open->close(); - } - } - - /** - * $rowBudget is deliberately unused: sample() is lazy and SchemaInferrer stops advancing it. + * $rowBudget is unused: SchemaInferrer hands each unit its remaining budget through sniffColumnTypes(). * - * @return Generator> + * @return Generator */ public function samples(int $rowBudget): iterable { foreach ($this->sources as $source) { - yield $this->sample($source); + yield new CSVFileSample($this->opener, $source); } } } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php new file mode 100644 index 0000000000..10bb522b1c --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php @@ -0,0 +1,56 @@ + + */ +final readonly class CSVFileSample implements IteratorAggregate, SniffsColumnTypes +{ + public function __construct( + private CSVSourceOpener $opener, + private SourceFile $source, + ) {} + + /** + * The source is opened on the first advance; abandoning the generator closes it. + * + * @return Generator + */ + public function getIterator(): Generator + { + $open = $this->opener->open($this->source); + + try { + yield from $open->records(); + } finally { + $open->close(); + } + } + + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes { + $open = $this->opener->open($this->source); + + try { + return $open->sniff($names, $rowBudget, $inference, $typer); + } finally { + $open->close(); + } + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php index 88d6011fa0..482134978c 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLineReader.php @@ -11,18 +11,23 @@ use function str_contains; use function str_starts_with; use function substr; -use function substr_count; final readonly class CSVLineReader { + private CSVRecordBoundary $boundary; + /** * @param null|int<1, max> $charactersReadInLine */ public function __construct( private string $enclosure, + string $separator = ',', + string $escape = '\\', private ?int $charactersReadInLine = null, private bool $removeBOM = true, - ) {} + ) { + $this->boundary = new CSVRecordBoundary($enclosure, $separator, $escape); + } /** * @return \Generator @@ -42,7 +47,7 @@ public function readLines(SourceStream $stream): Generator $lineNumber++; $buffer = ''; } else { - if ($this->isCompleteCSVRecord($buffer)) { + if ($this->boundary->isComplete($buffer)) { yield $this->removeBOM && $lineNumber === 0 ? $this->removeBOMFromLine(rtrim($buffer, "\r\n")) : rtrim($buffer, "\r\n"); @@ -61,19 +66,6 @@ public function readLines(SourceStream $stream): Generator } } - /** - * Check if the current buffer contains a complete CSV record - * by counting enclosures and ensuring they are properly paired. - */ - private function isCompleteCSVRecord(string $buffer): bool - { - if (!str_contains($buffer, $this->enclosure)) { - return true; - } - - return (substr_count($buffer, $this->enclosure) % 2) === 0; - } - /** * Remove Byte Order Mark (BOM) from the beginning of a line if present. */ diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php index cad701dc86..e030ac48df 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php @@ -5,51 +5,34 @@ namespace Flow\ETL\Adapter\CSV; use Flow\ETL\Row\RawRowValues; -use Flow\Filesystem\SourceStream; +use Flow\ETL\Schema\Inference\ColumnTypes; +use Flow\ETL\Schema\Inference\SchemaInference; +use Flow\Types\Type\TypeNarrower; use Generator; -final readonly class CSVOpenSource +interface CSVOpenSource { - public function __construct( - public SourceStream $stream, - public CSVDialect $dialect, - public CSVEncoder $encoder, - public CSVLineReader $lineReader, - ) {} - - public function close(): void - { - $this->stream->close(); - } + public function close(): void; /** * This instance is consumed afterwards. * * @return list */ - public function columns(): array - { - foreach ($this->lineReader->readLines($this->stream) as $line) { - $this->encoder->decode([$line]); - - break; - } - - return $this->encoder->headers() ?? []; - } + public function columns(): array; /** - * CSVLineReader::readLines() already joins a quoted multi-line record, so never re-split or re-join here. * This instance is consumed afterwards. * * @return Generator */ - public function records(): Generator - { - foreach ($this->lineReader->readLines($this->stream) as $line) { - foreach ($this->encoder->decode([$line]) as $values) { - yield $values; - } - } - } + public function records(): Generator; + + /** + * SchemaInferrer::sniff() over records(). This instance is consumed afterwards. + * + * @param list $names + * @param int<0, max>|-1 $rowBudget + */ + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes; } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php new file mode 100644 index 0000000000..b89f0de05f --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVRecordBoundary.php @@ -0,0 +1,56 @@ +pattern = + '/\A(?:' . $blanks . '(?:' . $enclosedField . '|' . $unenclosedField . ')' . $fieldEnd . ')*+\z/s'; + $this->scan = new CSVEnclosureScan($separator, $enclosure, $escape); + } + + public function isComplete(string $buffer): bool + { + if (!str_contains($buffer, $this->enclosure)) { + return true; + } + + $matched = preg_match($this->pattern, $buffer); + + // PCRE gives up at about 142k fields in one record; reading that as "incomplete" would glue every following line + return $matched === false ? $this->scan->endsOutsideAnEnclosure($buffer) : $matched === 1; + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php index 2bc90e9791..5e27c6390b 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSourceOpener.php @@ -28,9 +28,23 @@ public function open(SourceFile $source): CSVOpenSource $this->options->escape ?? $detected->escape, ); - return new CSVOpenSource( + if (NativeCSVOpenSource::isSupported()) { + return new NativeCSVOpenSource( + $stream, + new RustCSVReaderNative( + $dialect->separator, + $dialect->enclosure, + $dialect->escape, + $this->options->withHeader, + $this->options->emptyToNull, + $this->options->removeBOM, + ), + $this->options->charactersReadInLine, + ); + } + + return new PhpCSVOpenSource( $stream, - $dialect, new CSVEncoder( withHeader: $this->options->withHeader, separator: $dialect->separator, @@ -38,7 +52,13 @@ public function open(SourceFile $source): CSVOpenSource escape: $dialect->escape, emptyToNull: $this->options->emptyToNull, ), - new CSVLineReader($dialect->enclosure, $this->options->charactersReadInLine, $this->options->removeBOM), + new CSVLineReader( + $dialect->enclosure, + $dialect->separator, + $dialect->escape, + $this->options->charactersReadInLine, + $this->options->removeBOM, + ), ); } catch (Throwable $e) { $stream->close(); diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php new file mode 100644 index 0000000000..ab3da4a2c1 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/NativeCSVOpenSource.php @@ -0,0 +1,158 @@ + + */ + private int $chunkSize; + + /** + * @param null|int<1, max> $charactersReadInLine + */ + public function __construct( + private SourceStream $stream, + private RustCSVReaderNative $reader, + ?int $charactersReadInLine = null, + ) { + // a local file ignores the read length, as NativeLocalSourceStream::readLines() already does on the PHP path; + // for S3/Azure it is the range-request size - and from_csv() defaults it to 10 MB, which would be the chunk + $this->chunkSize = $stream instanceof NativeLocalSourceStream + ? self::CHUNK + : $charactersReadInLine ?? self::CHUNK; + } + + public static function isSupported(): bool + { + return extension_loaded('flow_php') && class_exists(RustCSVReaderNative::class, false); + } + + public function close(): void + { + $this->stream->close(); + } + + public function columns(): array + { + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + + $headers = $this->reader->headers(); + + if ($headers !== []) { + return $headers; + } + } + + $this->reader->finish(); + + return $this->reader->headers(); + } + + public function records(): Generator + { + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + + while (($batch = $this->reader->next(self::BATCH)) !== []) { + foreach ($batch as $values) { + yield $values; + } + } + } + + $this->reader->finish(); + + while (($batch = $this->reader->next(self::BATCH)) !== []) { + foreach ($batch as $values) { + yield $values; + } + } + } + + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes + { + // the Rust fold is StringTypeNarrower without its HTML and XML rungs - a DOMDocument per cell is not worth porting + if (!$typer instanceof StringTypeNarrower || $typer->emitsType(type_html()) || $typer->emitsType(type_xml())) { + return (new SchemaInferrer($inference, $typer))->sniff($names, $this->records(), $rowBudget); + } + + $fold = new RustColumnFoldNative($names, array_map( + static fn(Type $type): string => $type->toString(), + array_values(array_filter( + [ + type_json(), + type_uuid(), + type_float(), + type_integer(), + type_datetime(), + type_date(), + type_boolean(), + type_time_zone(), + ], + $typer->emitsType(...), + )), + )); + + foreach ($this->stream->iterate($this->chunkSize) as $chunk) { + $this->reader->feed($chunk); + $this->reader->fold($fold, $rowBudget === -1 ? -1 : max(0, $rowBudget - $fold->rows())); + + if ($rowBudget !== -1 && $fold->rows() >= $rowBudget) { + break; + } + } + + if ($rowBudget === -1 || $fold->rows() < $rowBudget) { + $this->reader->finish(); + $this->reader->fold($fold, $rowBudget === -1 ? -1 : max(0, $rowBudget - $fold->rows())); + } + + return ColumnTypes::fromColumnTypes( + array_map(TypeFactory::fromString(...), $fold->types()), + $fold->rows(), + $typer, + ); + } +} diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php new file mode 100644 index 0000000000..5af14d0595 --- /dev/null +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php @@ -0,0 +1,63 @@ +stream->close(); + } + + /** + * This instance is consumed afterwards. + * + * @return list + */ + public function columns(): array + { + foreach ($this->lineReader->readLines($this->stream) as $line) { + $this->encoder->decode([$line]); + + break; + } + + return $this->encoder->headers() ?? []; + } + + /** + * CSVLineReader::readLines() already joins a quoted multi-line record, so never re-split or re-join here. + * This instance is consumed afterwards. + * + * @return Generator + */ + public function records(): Generator + { + foreach ($this->lineReader->readLines($this->stream) as $line) { + foreach ($this->encoder->decode([$line]) as $values) { + yield $values; + } + } + } + + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes + { + return (new SchemaInferrer($inference, $typer))->sniff($names, $this->records(), $rowBudget); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php index 1c9f025979..c01f3cda5b 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php @@ -4,18 +4,43 @@ namespace Flow\ETL\Adapter\CSV\Tests\Context; +use Flow\ETL\Adapter\CSV\CSVDialect; +use Flow\ETL\Adapter\CSV\CSVEncoder; use Flow\ETL\Adapter\CSV\CSVFileReader; +use Flow\ETL\Adapter\CSV\CSVFileSample; +use Flow\ETL\Adapter\CSV\CSVLineReader; use Flow\ETL\Adapter\CSV\CSVOpenSource; use Flow\ETL\Adapter\CSV\CSVReadOptions; use Flow\ETL\Adapter\CSV\CSVSourceOpener; +use Flow\ETL\Adapter\CSV\NativeCSVOpenSource; +use Flow\ETL\Adapter\CSV\PhpCSVOpenSource; +use Flow\ETL\Adapter\CSV\RustCSVReaderNative; use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Row\RawRowValues; +use Flow\ETL\Schema; +use Flow\ETL\Schema\Inference\ColumnTypes; +use Flow\ETL\Schema\Inference\SchemaInference; +use Flow\ETL\Schema\Inference\SchemaInferrer; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; +use Flow\Filesystem\Local\MemoryFilesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path\Filter\OnlyFiles; +use Flow\Filesystem\SourceStream; +use Flow\Types\Type\Native\String\StringTypeNarrower; +use Flow\Types\Type\TypeNarrower; +use Generator; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; +use SplFileInfo; +use function Flow\ETL\Adapter\CSV\csv_detect_separator; +use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; +use function str_ends_with; +use function strlen; +use function substr; /** * Resolves the adapter's test fixtures and builds readers over them, so no test needs a private helper. @@ -47,6 +72,192 @@ public static function open( return (new CSVSourceOpener($filesystem, $options))->open(self::source($fixture)); } + /** + * Every *.csv under Fixtures/, keyed and valued by its path relative to Fixtures/. + * + * @return Generator + */ + public static function fixtures(): Generator + { + $root = self::path(''); + + /** @var SplFileInfo $file */ + foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator( + $root, + RecursiveDirectoryIterator::SKIP_DOTS, + )) as $file) { + if (str_ends_with($file->getFilename(), '.csv')) { + $fixture = substr($file->getPathname(), strlen($root)); + + yield $fixture => [$fixture]; + } + } + } + + /** + * The dialect CSVSourceOpener would resolve: pinned options first, detection for the rest. + */ + public static function dialect(SourceStream $stream, CSVReadOptions $options = new CSVReadOptions()): CSVDialect + { + $detected = csv_detect_separator($stream); + + return new CSVDialect( + $options->separator ?? $detected->separator, + $options->enclosure ?? $detected->enclosure, + $options->escape ?? $detected->escape, + ); + } + + /** + * Requires the extension. + */ + public static function openNative( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): NativeCSVOpenSource { + return self::openNativeStream($filesystem->readFrom(path_real(self::path($fixture))), $options); + } + + /** + * The native open source over an already-open stream - requires the extension. + */ + public static function openNativeStream( + SourceStream $stream, + CSVReadOptions $options = new CSVReadOptions(), + ): NativeCSVOpenSource { + $dialect = self::dialect($stream, $options); + + return new NativeCSVOpenSource( + $stream, + new RustCSVReaderNative( + $dialect->separator, + $dialect->enclosure, + $dialect->escape, + $options->withHeader, + $options->emptyToNull, + $options->removeBOM, + ), + $options->charactersReadInLine, + ); + } + + public static function openPhp( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): PhpCSVOpenSource { + $stream = $filesystem->readFrom(path_real(self::path($fixture))); + $dialect = self::dialect($stream, $options); + + return new PhpCSVOpenSource( + $stream, + new CSVEncoder( + withHeader: $options->withHeader, + separator: $dialect->separator, + enclosure: $dialect->enclosure, + escape: $dialect->escape, + emptyToNull: $options->emptyToNull, + ), + new CSVLineReader( + $dialect->enclosure, + $dialect->separator, + $dialect->escape, + $options->charactersReadInLine, + $options->removeBOM, + ), + ); + } + + /** + * Every record's values and metadata, for strict comparison across paths - consumes the source. + * + * @return list, array}> + */ + public static function records(CSVOpenSource $open): array + { + $records = []; + + foreach ($open->records() as $record) { + $records[] = [$record->values, $record->metadata]; + } + + return $records; + } + + /** + * The schema CSVExtractor infers - through CSVFileReader::samples(), native when the extension is loaded. + */ + public static function infer(SchemaInference $inference, string ...$fixtures): Schema + { + $reader = self::reader(sources: self::sources(...$fixtures)); + + return (new SchemaInferrer($inference, new StringTypeNarrower($inference->candidates()->toArray())))->infer( + $reader->header()->names, + $reader->samples($inference->sampleSize), + ); + } + + /** + * The canonical PHP fold: the PHP open source's records observed row by row, whatever the opener would pick. + */ + public static function inferPhp(SchemaInference $inference, string ...$fixtures): Schema + { + $names = []; + + foreach ($fixtures as $fixture) { + $open = self::openPhp($fixture); + $names = $open->columns(); + $open->close(); + + if ($names !== []) { + break; + } + } + + $samples = []; + + foreach ($fixtures as $fixture) { + $samples[] = self::phpRecords($fixture); + } + + return (new SchemaInferrer($inference, new StringTypeNarrower($inference->candidates()->toArray())))->infer( + $names, + $samples, + ); + } + + /** + * The PHP open source's records, opened on the first advance and closed when abandoned - a SchemaSampler unit. + * + * @return Generator + */ + public static function phpRecords(string $fixture): Generator + { + $open = self::openPhp($fixture); + + try { + yield from $open->records(); + } finally { + $open->close(); + } + } + + public static function memory(string $content): MemoryFilesystem + { + $filesystem = memory_filesystem(); + $stream = $filesystem->writeTo(self::memorySource()->path); + $stream->append($content); + $stream->close(); + + return $filesystem; + } + + public static function memorySource(): SourceFile + { + return new SourceFile(path('memory://source.csv')); + } + public static function path(string $fixture): string { return __DIR__ . '/../Fixtures/' . $fixture; @@ -66,6 +277,38 @@ public static function reader( return new CSVFileReader(new CSVSourceOpener($filesystem, $options), $sources); } + public static function sample( + string $fixture, + Filesystem $filesystem = new NativeLocalFilesystem(), + CSVReadOptions $options = new CSVReadOptions(), + ): CSVFileSample { + return new CSVFileSample(new CSVSourceOpener($filesystem, $options), self::source($fixture)); + } + + /** + * The fixture's partial ColumnTypes observed row by row over the PHP records, and sniffed by CSVFileSample - + * natively when the extension is loaded. + * + * @param int<0, max>|-1 $rowBudget + * + * @return array{ColumnTypes, ColumnTypes} + */ + public static function sniffBothWays( + string $fixture, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): array { + $open = self::openPhp($fixture); + $names = $open->columns(); + $open->close(); + + return [ + (new SchemaInferrer($inference, $typer))->sniff($names, self::phpRecords($fixture), $rowBudget), + self::sample($fixture)->sniffColumnTypes($names, $rowBudget, $inference, $typer), + ]; + } + public static function source(string $fixture): SourceFile { return new SourceFile(path_real(self::path($fixture))); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php new file mode 100644 index 0000000000..c1483b5198 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVRecordBoundaryContext.php @@ -0,0 +1,38 @@ + + */ + public static function buffers(): Generator + { + yield 'no enclosure' => [',', '"', '\\', 'a,b', true]; + yield 'closed enclosure' => [',', '"', '\\', '"a",b', true]; + yield 'open enclosure' => [',', '"', '\\', '"a', false]; + yield 'open enclosure across a line' => [',', '"', '\\', "\"a\nb", false]; + yield 'closed across a line' => [',', '"', '\\', "\"a\nb\",c", true]; + yield 'escaped enclosure keeps it open' => [',', '"', '\\', '"x\"y', false]; + yield 'escape as the last byte' => [',', '"', '\\', '"x\\', false]; + yield 'doubled enclosure keeps it open' => [',', '"', '\\', '"a""', false]; + yield 'doubled then closed' => [',', '"', '\\', '"a"""', true]; + yield 'enclosure inside an unenclosed field' => [',', '"', '\\', 'x"y,1', true]; + yield 'blanks before an opening enclosure' => [',', '"', '\\', "a, \t\"b", false]; + yield 'carriage return before an opening enclosure' => [',', '"', '\\', "a,\r\"b", false]; + yield 'junk after a closing enclosure' => [',', '"', '\\', '"a"x"y', true]; + yield 'empty escape' => [',', '"', '', '"x\",1', true]; + yield 'escape equal to the enclosure' => [',', '"', '"', '"a""', false]; + yield 'custom separator and enclosure' => [';', "'", '\\', "x,'y;1", true]; + yield 'custom enclosure left open' => [';', "'", '\\', "a;'b", false]; + yield 'whitespace separator is not a blank' => ["\t", '"', '\\', "a\t\t\"b", false]; + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php new file mode 100644 index 0000000000..694a6836b1 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingFilesystem.php @@ -0,0 +1,84 @@ +wrapped->appendTo($path); + } + + public function getSystemTmpDir(): Path + { + return $this->wrapped->getSystemTmpDir(); + } + + public function list(Path $path, Filter $pathFilter = new KeepAll()): Generator + { + yield from $this->wrapped->list($path, $pathFilter); + } + + public function mount(): Mount + { + return $this->wrapped->mount(); + } + + public function mv(Path $from, Path $to): bool + { + return $this->wrapped->mv($from, $to); + } + + public function readFrom(Path $path): SourceStream + { + $stream = $this->wrapped->readFrom($path); + + return $this->lastStream = new LengthCapturingSourceStream( + $stream->read(max(1, (int) $stream->size()), 0), + $path, + ); + } + + public function rm(Path $path): bool + { + return $this->wrapped->rm($path); + } + + public function status(Path $path): ?FileStatus + { + return $this->wrapped->status($path); + } + + public function supports(Path $path): bool + { + return $this->wrapped->supports($path); + } + + public function writeTo(Path $path): DestinationStream + { + return $this->wrapped->writeTo($path); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php index 008c96f52b..a7298f5536 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Double/LengthCapturingSourceStream.php @@ -19,6 +19,11 @@ final class LengthCapturingSourceStream implements SourceStream */ public array $capturedLengths = []; + /** + * @var list the length argument captured on each iterate() call + */ + public array $capturedIterateLengths = []; + public function __construct( private readonly string $contents, private readonly Path $path, @@ -38,6 +43,8 @@ public function isOpen(): bool public function iterate(int $length = 1): Generator { + $this->capturedIterateLengths[] = $length; + for ($i = 0; $i < strlen($this->contents); $i += $length) { yield substr($this->contents, $i, $length); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv new file mode 100644 index 0000000000..047e70a5fc --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/bare_quote_then_row.csv @@ -0,0 +1,3 @@ +a,b +x"y,1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv new file mode 100644 index 0000000000..398a0836db --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/doubled_quote_then_row.csv @@ -0,0 +1,2 @@ +"a""b",1 +"c",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv new file mode 100644 index 0000000000..004cbefa36 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_no_escape_char.csv @@ -0,0 +1,3 @@ +a,b +"x\"y",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv new file mode 100644 index 0000000000..004cbefa36 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_then_row.csv @@ -0,0 +1,3 @@ +a,b +"x\"y",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv new file mode 100644 index 0000000000..5529c19a29 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_newline.csv @@ -0,0 +1,4 @@ +a,b +"x\"y +z",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv new file mode 100644 index 0000000000..e83306b1ce --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/escaped_quote_with_separator.csv @@ -0,0 +1,3 @@ +a,b +"x\"y,z",1 +"p",2 diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv new file mode 100644 index 0000000000..8d72765c7c --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Fixtures/fold_traps.csv @@ -0,0 +1,4 @@ +id,pad,n,dup,dup +1, 5 ,NULL,1,x +2,7,nil,2,y +3,8,5,3,z diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php index af2d2932f4..b9a4404a74 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php @@ -22,6 +22,7 @@ use Flow\Filesystem\Tests\OperatingSystem; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\DataProviderExternal; use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; @@ -30,6 +31,8 @@ use function array_map; use function array_sum; use function count; +use function fclose; +use function fgetcsv; use function Flow\ETL\Adapter\CSV\from_csv; use function Flow\ETL\Adapter\CSV\to_csv; use function Flow\ETL\DSL\config; @@ -46,9 +49,14 @@ use function Flow\ETL\DSL\schema_metadata; use function Flow\ETL\DSL\schema_to_ascii; use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\native_local_filesystem; use function Flow\Filesystem\DSL\path_real; +use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_html; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_xml; +use function fopen; use function iterator_to_array; use function max; use function sort; @@ -1126,4 +1134,139 @@ private function ensureBOMExists(string $path, string $BOM): bool return $contents === $BOM; } + + public function test_an_escaped_quote_does_not_glue_the_following_record(): void + { + static::assertSame( + [['a' => 'x\"y', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_then_row.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_a_bare_quote_in_an_unenclosed_field_does_not_glue_the_following_record(): void + { + static::assertSame( + [['a' => 'x"y', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('bare_quote_then_row.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_next_to_an_embedded_newline_keeps_one_record(): void + { + static::assertSame( + [['a' => "x\\\"y\nz", 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_with_newline.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_next_to_an_embedded_separator_keeps_one_record(): void + { + static::assertSame( + [['a' => 'x\"y,z', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_with_separator.csv'))) + ->fetch() + ->toArray(), + ); + } + + public function test_an_escaped_quote_read_without_an_escape_character_closes_the_field(): void + { + static::assertSame( + [['a' => 'x\y"', 'b' => 1], ['a' => 'p', 'b' => 2]], + df() + ->read(from_csv(CSVFixtureContext::path('escaped_quote_no_escape_char.csv'))->withEscape('')) + ->fetch() + ->toArray(), + ); + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_record_counts_match_fgetcsv_for_every_fixture(string $fixture): void + { + $path = CSVFixtureContext::path($fixture); + $stream = native_local_filesystem()->readFrom(path_real($path)); + $dialect = CSVFixtureContext::dialect($stream); + $stream->close(); + + $handle = fopen($path, 'rb'); + $records = 0; + + while (fgetcsv($handle, 0, $dialect->separator, $dialect->enclosure, $dialect->escape) !== false) { + $records++; + } + + fclose($handle); + + static::assertSame( + max(0, $records - 1), + df() + ->read( + from_csv($path) + ->withSeparator($dialect->separator) + ->withEnclosure($dialect->enclosure) + ->withEscape($dialect->escape), + ) + ->fetch() + ->count(), + ); + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_native_and_php_inference_agree_on_every_fixture(string $fixture): void + { + static::assertEquals( + CSVFixtureContext::inferPhp(infer_schema()->build(), $fixture), + CSVFixtureContext::infer(infer_schema()->build(), $fixture), + ); + } + + public function test_sample_size_minus_one_agrees_on_both_paths(): void + { + $inference = infer_schema()->sampleSize(-1)->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } + + public function test_union_by_name_over_a_glob_agrees_on_both_paths(): void + { + $inference = infer_schema()->unionByName()->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'columns_diverge/a.csv', 'columns_diverge/b.csv'), + CSVFixtureContext::infer($inference, 'columns_diverge/a.csv', 'columns_diverge/b.csv'), + ); + } + + public function test_a_restricted_candidate_set_agrees_on_both_paths(): void + { + $inference = infer_schema()->types(type_integer(), type_boolean())->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } + + public function test_html_or_xml_candidates_fall_back_to_the_php_path(): void + { + $inference = infer_schema()->types(type_html(), type_xml(), type_integer())->build(); + + static::assertEquals( + CSVFixtureContext::inferPhp($inference, 'orders_flow.csv'), + CSVFixtureContext::infer($inference, 'orders_flow.csv'), + ); + } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php index 565eefe3cb..6041f355ce 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php @@ -10,7 +10,6 @@ use Flow\Filesystem\Local\NativeLocalFilesystem; use PHPUnit\Framework\Attributes\TestWith; -use function array_keys; use function count; use function iterator_to_array; @@ -110,31 +109,6 @@ public function test_header_stops_at_the_first_source_with_a_header(): void static::assertStringEndsWith('a.csv', $header->source); } - public function test_sample_closes_its_stream_when_abandoned(): void - { - $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $sample = CSVFixtureContext::reader($counting)->sample(CSVFixtureContext::source('five_rows.csv')); - - $sample->current(); - unset($sample); - - static::assertSame(1, $counting->closedStreams()); - } - - public function test_sample_yields_every_row_of_a_source(): void - { - $records = iterator_to_array( - CSVFixtureContext::reader()->sample(CSVFixtureContext::source('ragged.csv')), - false, - ); - - static::assertCount(3, $records); - - foreach ($records as $record) { - static::assertSame(['id', 'name', 'v'], array_keys($record->values)); - } - } - public function test_samples_does_not_ration_the_row_budget(): void { $units = iterator_to_array( @@ -160,7 +134,7 @@ public function test_samples_walks_the_sources_in_listing_order(): void static::assertSame([['id' => '1', 'name' => 'a'], ['id' => '3', 'label' => 'c']], $first); } - public function test_samples_yields_one_unstarted_generator_per_source(): void + public function test_samples_yields_one_unstarted_unit_per_source(): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); $units = iterator_to_array( @@ -170,12 +144,12 @@ public function test_samples_yields_one_unstarted_generator_per_source(): void $beforeAdvancing = $counting->readFromCalls; - $units[0]->current(); + $units[0]->getIterator()->current(); $afterAdvancingOne = $counting->readFromCalls; static::assertCount(2, $units); - static::assertSame(0, $beforeAdvancing, 'samples() yields UNSTARTED generators'); + static::assertSame(0, $beforeAdvancing, 'samples() yields UNSTARTED units'); static::assertSame(1, $afterAdvancingOne, 'advancing the first unit opens exactly one source'); } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php new file mode 100644 index 0000000000..c20e5b80ba --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php @@ -0,0 +1,93 @@ +values)); + } + } + + public function test_an_abandoned_iteration_closes_its_stream(): void + { + $counting = new CountingFilesystem(new NativeLocalFilesystem()); + $iterator = CSVFixtureContext::sample('five_rows.csv', $counting)->getIterator(); + + $iterator->current(); + unset($iterator); + + static::assertSame(1, $counting->closedStreams()); + } + + /** + * @param int<0, max>|-1 $rowBudget + */ + #[TestWith([-1, false])] + #[TestWith([2, false])] + #[TestWith([100, false])] + #[TestWith([-1, true])] + public function test_sniffing_equals_observing_its_records(int $rowBudget, bool $htmlCandidate): void + { + $inference = $htmlCandidate + ? infer_schema()->types(type_html(), type_integer())->build() + : infer_schema()->build(); + + [$observed, $sniffed] = CSVFixtureContext::sniffBothWays( + 'orders_flow.csv', + $rowBudget, + $inference, + new StringTypeNarrower($inference->candidates()->toArray()), + ); + + static::assertSame($observed->rows(), $sniffed->rows()); + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } + + public function test_sniffing_narrows_with_the_narrower_it_is_handed(): void + { + $inference = infer_schema()->build(); + [$observed, $sniffed] = CSVFixtureContext::sniffBothWays( + 'orders_flow.csv', + -1, + $inference, + new StringTypeNarrower([type_string()]), + ); + + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php index eefee60676..8412bd6536 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php @@ -20,7 +20,7 @@ public function test_reading_csv_with_custom_character_limit(): void $path = __DIR__ . '/../Fixtures/more_than_1000_characters_per_line.csv'; $stream = NativeLocalSourceStream::open(path_real($path)); - $reader = new CSVLineReader('"', 2000); + $reader = new CSVLineReader('"', charactersReadInLine: 2000); $lines = iterator_to_array($reader->readLines($stream)); static::assertCount(2, $lines); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php index 483ac1c6b3..ca2cce5379 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSourceOpenerTest.php @@ -23,7 +23,6 @@ public function test_a_pinned_separator_wins_over_detection(): void ))->open(CSVFixtureContext::source('semicolon.csv')); try { - static::assertSame(',', $open->dialect->separator); static::assertSame(['id;name'], $open->columns()); } finally { $open->close(); @@ -69,7 +68,7 @@ public function test_open_detects_the_dialect(): void ))->open(CSVFixtureContext::source('semicolon.csv')); try { - static::assertSame(';', $open->dialect->separator); + static::assertSame(['id', 'name'], $open->columns()); } finally { $open->close(); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php new file mode 100644 index 0000000000..34bd1d36b6 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/NativeCSVOpenSourceTest.php @@ -0,0 +1,335 @@ +close(); + + static::assertSame(1, $counting->closedStreams()); + } + + public function test_columns_of_an_empty_source_is_empty(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('empty.csv'); + + try { + static::assertSame([], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_reads_a_quoted_multiline_header_as_one_record(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('multiline_header.csv'); + + try { + static::assertSame(['id', "na\nme"], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_reports_the_resolved_header(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('header_only.csv'); + + try { + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_columns_without_a_header_line_are_generated(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('two_columns.csv', options: new CSVReadOptions(withHeader: false)); + + try { + static::assertSame(['e00', 'e01'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_records_yields_every_row_with_the_full_key_set(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('ragged.csv'); + + try { + $records = iterator_to_array($open->records(), false); + + static::assertCount(3, $records); + + foreach ($records as $record) { + static::assertSame(['id', 'name', 'v'], array_keys($record->values)); + } + } finally { + $open->close(); + } + } + + public function test_records_yields_one_logical_record_per_iteration(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('multiline_strings.csv'); + + try { + $records = iterator_to_array($open->records(), false); + + static::assertNotSame([], $records); + static::assertStringContainsString("\n", implode('', array_map('strval', $records[0]->values))); + } finally { + $open->close(); + } + } + + #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + public function test_native_and_php_open_sources_yield_identical_records(string $fixture): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp($fixture); + $native = CSVFixtureContext::openNative($fixture); + + try { + static::assertSame(CSVFixtureContext::records($php), CSVFixtureContext::records($native)); + } finally { + $php->close(); + $native->close(); + } + } + + /** + * @param positive-int $chunkSize + */ + #[TestWith([1])] + #[TestWith([7])] + #[TestWith([4096])] + public function test_records_are_identical_across_chunk_sizes(int $chunkSize): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $contents = (string) file_get_contents(CSVFixtureContext::path('multiline_strings.csv')); + + static::assertSame( + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )), + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + (new CSVReadOptions())->withCharactersReadInLine($chunkSize), + )), + ); + } + + public function test_a_remote_stream_is_read_in_steps_of_characters_read_in_line(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $stream = new LengthCapturingSourceStream("id,name\n1,a\n", path('s3://bucket/a.csv')); + + CSVFixtureContext::records(CSVFixtureContext::openNativeStream( + $stream, + (new CSVReadOptions())->withCharactersReadInLine(4096), + )); + + static::assertSame([4096], $stream->capturedIterateLengths); + } + + public function test_a_remote_stream_without_characters_read_in_line_is_read_in_default_chunks(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $stream = new LengthCapturingSourceStream("id,name\n1,a\n", path('s3://bucket/a.csv')); + + CSVFixtureContext::records(CSVFixtureContext::openNativeStream($stream)); + + static::assertSame([NativeCSVOpenSource::CHUNK], $stream->capturedIterateLengths); + } + + #[TestWith(['multiline_header.csv'])] + #[TestWith(['with_utf8_bom.csv'])] + #[TestWith(['semicolon.csv'])] + #[TestWith(['single_quotes_csv.csv'])] + public function test_columns_resolves_the_same_header_on_both_paths(string $fixture): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp($fixture); + $native = CSVFixtureContext::openNative($fixture); + + try { + static::assertSame($php->columns(), $native->columns()); + } finally { + $php->close(); + $native->close(); + } + } + + public function test_a_zero_byte_source_resolves_no_header_on_both_paths(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $php = CSVFixtureContext::openPhp('empty.csv'); + $native = CSVFixtureContext::openNative('empty.csv'); + + try { + static::assertSame([], $php->columns()); + static::assertSame([], $native->columns()); + } finally { + $php->close(); + $native->close(); + } + } + + public function test_is_supported_is_false_without_the_extension(): void + { + if (class_exists(RustCSVReaderNative::class, false)) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is loaded'); + } + + static::assertFalse(NativeCSVOpenSource::isSupported()); + } + + /** + * @param int<0, max>|-1 $rowBudget + */ + #[TestWith([2, 2])] + #[TestWith([-1, 5])] + #[TestWith([10, 5])] + public function test_sniff_folds_at_most_the_row_budget(int $rowBudget, int $rows): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $open = CSVFixtureContext::openNative('five_rows.csv'); + + try { + static::assertSame( + $rows, + $open->sniff( + ['id', 'name'], + $rowBudget, + infer_schema()->build(), + new StringTypeNarrower(InferredTypes::default()->toArray()), + )->rows(), + ); + } finally { + $open->close(); + } + } + + public function test_sniff_folds_date_and_time_zone_columns_like_observing_the_records(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + $contents = "at,day,zone,n\n2024-01-01 10:00:00,2024-01-01,Europe/Warsaw,\n2024-02-02 11:00:00,2024-02-02,+02:00,1\n"; + $inference = infer_schema()->build(); + $typer = new StringTypeNarrower($inference->candidates()->toArray()); + $names = ['at', 'day', 'zone', 'n']; + $observed = (new SchemaInferrer($inference, $typer))->sniff( + $names, + CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )->records(), + -1, + ); + $sniffed = CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream($contents, path('s3://bucket/a.csv')), + )->sniff($names, -1, $inference, $typer); + + static::assertEquals( + $observed->schema(new TypeFloor($inference->candidates())), + $sniffed->schema(new TypeFloor($inference->candidates())), + ); + } + + public function test_columns_of_a_header_without_a_trailing_newline(): void + { + if (!NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is not loaded'); + } + + static::assertSame( + ['id', 'name'], + CSVFixtureContext::openNativeStream( + new LengthCapturingSourceStream('id,name', path('s3://bucket/a.csv')), + )->columns(), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php similarity index 81% rename from src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php rename to src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php index 6858360697..debe8ca9b9 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVOpenSourceTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/PhpCSVOpenSourceTest.php @@ -15,12 +15,12 @@ use function implode; use function iterator_to_array; -final class CSVOpenSourceTest extends FlowTestCase +final class PhpCSVOpenSourceTest extends FlowTestCase { public function test_close_closes_the_stream_once(): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $open = CSVFixtureContext::open('two_rows.csv', $counting); + $open = CSVFixtureContext::openPhp('two_rows.csv', $counting); $open->close(); @@ -29,7 +29,7 @@ public function test_close_closes_the_stream_once(): void public function test_columns_of_an_empty_source_is_empty(): void { - $open = CSVFixtureContext::open('empty.csv'); + $open = CSVFixtureContext::openPhp('empty.csv'); try { static::assertSame([], $open->columns()); @@ -40,7 +40,7 @@ public function test_columns_of_an_empty_source_is_empty(): void public function test_columns_reads_a_quoted_multiline_header_as_one_record(): void { - $open = CSVFixtureContext::open('multiline_header.csv'); + $open = CSVFixtureContext::openPhp('multiline_header.csv'); try { static::assertSame(['id', "na\nme"], $open->columns()); @@ -51,7 +51,7 @@ public function test_columns_reads_a_quoted_multiline_header_as_one_record(): vo public function test_columns_reports_the_resolved_header(): void { - $open = CSVFixtureContext::open('header_only.csv'); + $open = CSVFixtureContext::openPhp('header_only.csv'); try { static::assertSame(['id', 'name'], $open->columns()); @@ -62,7 +62,7 @@ public function test_columns_reports_the_resolved_header(): void public function test_columns_without_a_header_line_are_generated(): void { - $open = CSVFixtureContext::open('two_columns.csv', options: new CSVReadOptions(withHeader: false)); + $open = CSVFixtureContext::openPhp('two_columns.csv', options: new CSVReadOptions(withHeader: false)); try { static::assertSame(['e00', 'e01'], $open->columns()); @@ -73,7 +73,7 @@ public function test_columns_without_a_header_line_are_generated(): void public function test_records_yields_every_row_with_the_full_key_set(): void { - $open = CSVFixtureContext::open('ragged.csv'); + $open = CSVFixtureContext::openPhp('ragged.csv'); try { $records = iterator_to_array($open->records(), false); @@ -90,7 +90,7 @@ public function test_records_yields_every_row_with_the_full_key_set(): void public function test_records_yields_one_logical_record_per_iteration(): void { - $open = CSVFixtureContext::open('multiline_strings.csv'); + $open = CSVFixtureContext::openPhp('multiline_strings.csv'); try { $records = iterator_to_array($open->records(), false); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php new file mode 100644 index 0000000000..7864e1ccb3 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVEnclosureScanTest.php @@ -0,0 +1,27 @@ +endsOutsideAnEnclosure($buffer), + ); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php index 0ce9fce2aa..ae7675d4b2 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php @@ -7,17 +7,167 @@ use Flow\ETL\Adapter\CSV\CSVLineReader; use Flow\ETL\Adapter\CSV\Tests\Double\LengthCapturingSourceStream; use Flow\Filesystem\Stream\MemorySourceStream; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use function Flow\Filesystem\DSL\path; +use function iterator_to_array; +use function preg_last_error; +use function str_repeat; + +use const PREG_NO_ERROR; final class CSVLineReaderTest extends TestCase { + /** + * @return Generator}> + */ + public static function record_boundaries(): Generator + { + yield 'ends outside an enclosure' => [',', '"', '\\', "\"a\",b\nc,d", ['"a",b', 'c,d']]; + yield 'ends inside an enclosure' => [',', '"', '\\', "\"a\nb\",c\nd", ["\"a\nb\",c", 'd']]; + yield 'escaped enclosure' => [',', '"', '\\', "\"x\\\"y\nz\",1\n\"p\",2", ["\"x\\\"y\nz\",1", '"p",2']]; + yield 'escape before the line end' => [',', '"', '\\', "\"x\\\n\",1\n\"p\",2", ["\"x\\\n\",1", '"p",2']]; + yield 'doubled enclosure' => [',', '"', '\\', "\"a\"\"\nb\",1\n\"c\",2", ["\"a\"\"\nb\",1", '"c",2']]; + yield 'doubled enclosure at the buffer end' => [',', '"', '\\', "\"a\"\"\n\",1", ["\"a\"\"\n\",1"]]; + yield 'enclosure inside an unenclosed field' => [',', '"', '\\', "x\"y,1\n\"p\",2", ['x"y,1', '"p",2']]; + yield 'blanks before an opening enclosure' => [',', '"', '\\', "a, \t\"b\nc\"\nd", ["a, \t\"b\nc\"", 'd']]; + yield 'junk after a closing enclosure' => [',', '"', '\\', "\"a\"x\"y,1\nb", ['"a"x"y,1', 'b']]; + yield 'empty escape' => [',', '"', '', "\"x\\\",1\n\"p\",2", ['"x\\",1', '"p",2']]; + yield 'escape equal to the enclosure' => [',', '"', '"', "\"a\"\"\nb\",1\nc", ["\"a\"\"\nb\",1", 'c']]; + yield 'custom separator and enclosure' => [ + ';', + "'", + '\\', + "x,'y;1\na;'b\nc';d\ne;f", + ["x,'y;1", "a;'b\nc';d", 'e;f'], + ]; + } + + public function test_a_record_ending_outside_an_enclosure_is_complete(): void + { + static::assertSame( + ['"a",b', 'c,d'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\",b\nc,d"))), + ); + } + + public function test_a_record_ending_inside_an_enclosure_is_incomplete(): void + { + static::assertSame( + ["\"a\nb\",c", 'd'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\nb\",c\nd"))), + ); + } + + public function test_an_escaped_enclosure_does_not_close_the_record(): void + { + static::assertSame( + ["\"x\\\"y\nz\",1", '"p",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"x\\\"y\nz\",1\n\"p\",2"))), + ); + } + + public function test_a_doubled_enclosure_does_not_close_the_record(): void + { + static::assertSame( + ["\"a\"\"\nb\",1", '"c",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("\"a\"\"\nb\",1\n\"c\",2"))), + ); + } + + public function test_an_enclosure_inside_an_unenclosed_field_is_a_literal_byte(): void + { + static::assertSame( + ['x"y,1', '"p",2'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("x\"y,1\n\"p\",2"))), + ); + } + + public function test_an_empty_escape_makes_the_escape_character_ordinary(): void + { + $content = "\"x\\\",1\n\"p\",2"; + + static::assertSame( + ["\"x\\\",1\n\"p\",2"], + iterator_to_array((new CSVLineReader('"', escape: '\\'))->readLines(new MemorySourceStream($content))), + ); + static::assertSame( + ['"x\\",1', '"p",2'], + iterator_to_array((new CSVLineReader('"', escape: ''))->readLines(new MemorySourceStream($content))), + ); + } + + public function test_a_buffer_without_any_enclosure_is_complete(): void + { + static::assertSame( + ['a,b', 'c,d'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream("a,b\nc,d"))), + ); + } + + public function test_a_custom_separator_and_enclosure_are_honoured(): void + { + static::assertSame( + ["x,'y;1", "a;'b\nc';d", 'e;f'], + iterator_to_array((new CSVLineReader("'", ';'))->readLines( + new MemorySourceStream("x,'y;1\na;'b\nc';d\ne;f"), + )), + ); + } + + public function test_a_buffer_of_one_megabyte_without_a_closing_enclosure_does_not_blow_the_pcre_backtrack_limit(): void + { + $open = '"' . str_repeat('ab,\\"""c', 1 << 17); + + static::assertSame( + [$open . "\nd"], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($open . "\nd"))), + ); + static::assertSame(PREG_NO_ERROR, preg_last_error()); + } + + public function test_a_record_with_more_fields_than_pcre_can_match_is_still_split_at_its_end(): void + { + $closed = str_repeat('"a",', 200_000) . '"b"'; + $open = str_repeat('"a",', 200_000) . '"b'; + + static::assertSame( + [$closed, 'c'], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($closed . "\nc"))), + ); + static::assertSame( + [$open . "\nc\""], + iterator_to_array((new CSVLineReader('"'))->readLines(new MemorySourceStream($open . "\nc\""))), + ); + } + + /** + * @param non-empty-string $content + * @param list $expected + */ + #[DataProvider('record_boundaries')] + public function test_the_pattern_splits_records_at_their_end( + string $separator, + string $enclosure, + string $escape, + string $content, + array $expected, + ): void { + static::assertSame( + $expected, + iterator_to_array((new CSVLineReader($enclosure, $separator, $escape))->readLines( + new MemorySourceStream($content), + )), + ); + } + public function test_characters_read_in_line_is_passed_through_to_the_stream(): void { $stream = new LengthCapturingSourceStream("id,name\n1,foo", path('s3://bucket/users.csv')); - iterator_to_array((new CSVLineReader('"', 4096))->readLines($stream)); + iterator_to_array((new CSVLineReader('"', charactersReadInLine: 4096))->readLines($stream)); static::assertSame([4096], $stream->capturedLengths); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php new file mode 100644 index 0000000000..2c4dfbea75 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVRecordBoundaryTest.php @@ -0,0 +1,34 @@ +isComplete($buffer)); + } + + public function test_a_record_with_more_fields_than_pcre_can_match_is_decided_by_the_scan(): void + { + $boundary = new CSVRecordBoundary('"'); + + static::assertTrue($boundary->isComplete(str_repeat('"a",', 200_000) . '"b"')); + static::assertFalse($boundary->isComplete(str_repeat('"a",', 200_000) . '"b')); + } +} diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php new file mode 100644 index 0000000000..6ad2c73203 --- /dev/null +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSourceOpenerTest.php @@ -0,0 +1,87 @@ +open( + CSVFixtureContext::memorySource(), + ); + + try { + static::assertInstanceOf(NativeCSVOpenSource::class, $open); + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_open_returns_the_php_source_when_the_extension_is_absent(): void + { + if (NativeCSVOpenSource::isSupported()) { + static::markTestSkipped('flow_php extension with RustCSVReaderNative is loaded'); + } + + $open = (new CSVSourceOpener(CSVFixtureContext::memory("id,name\n1,a\n"), new CSVReadOptions()))->open( + CSVFixtureContext::memorySource(), + ); + + try { + static::assertInstanceOf(PhpCSVOpenSource::class, $open); + static::assertSame(['id', 'name'], $open->columns()); + } finally { + $open->close(); + } + } + + public function test_a_failing_detection_closes_the_stream_on_both_paths(): void + { + $throwing = new ThrowingSourceFilesystem(CSVFixtureContext::memory("id,name\n1,a\n")); + + $this->expectException(RuntimeException::class); + + try { + (new CSVSourceOpener($throwing, new CSVReadOptions()))->open(CSVFixtureContext::memorySource()); + } finally { + static::assertTrue($throwing->lastStream?->closed, 'open() closes the stream it opened before rethrowing'); + } + } + + public function test_characters_read_in_line_reaches_the_opened_source(): void + { + $filesystem = new LengthCapturingFilesystem(CSVFixtureContext::memory("id,name\n1,a\n")); + $open = (new CSVSourceOpener($filesystem, (new CSVReadOptions())->withCharactersReadInLine(4096)))->open( + CSVFixtureContext::memorySource(), + ); + + try { + iterator_to_array($open->records()); + } finally { + $open->close(); + } + + NativeCSVOpenSource::isSupported() + ? static::assertSame([4096], $filesystem->lastStream?->capturedIterateLengths) + : static::assertSame([null, 4096], $filesystem->lastStream?->capturedLengths); + } +} diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php index adecc32882..fee2112843 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php @@ -46,6 +46,24 @@ public function __construct( } } + /** + * A fold computed elsewhere (a native reader, a subprocess) - equal to the observe() fold over the same rows. + * + * @param array> $types - first-seen order, header names first + */ + public static function fromColumnTypes( + array $types, + int $rows, + TypeNarrower $typer, + TypeWidener $widener = new TypeWidener(), + ): self { + $columns = new self([], $typer, $widener); + $columns->types = $types; + $columns->rows = $rows; + + return $columns; + } + /** * Merge left-to-right in listing order: name order follows first-seen, so a different bracketing of the name * sequence changes the definition order (never the types). diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php index 12e9383b06..9bc005ea82 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php @@ -40,11 +40,15 @@ public function infer(array $names, iterable $sources): Schema break; } - $partial = $this->sniff( - $names, - $source, - $this->inference->sampleSize === -1 ? -1 : max(0, $this->inference->sampleSize - $columns->rows()), - ); + $rowBudget = $this->inference->sampleSize === -1 + ? -1 + : max(0, $this->inference->sampleSize - $columns->rows()); + + if ($source instanceof SniffsColumnTypes) { + $partial = $source->sniffColumnTypes($names, $rowBudget, $this->inference, $this->typer); + } else { + $partial = $this->sniff($names, $source, $rowBudget); + } if ($partial->rows() > 0) { $sniffed++; diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php index 567c2c9ff9..d7224f93c1 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php @@ -10,9 +10,8 @@ interface SchemaSampler { /** * One inner iterable per sampling unit (a file, a byte-range chunk, a sheet range), in listing order. - * The outer iterable is exactly what SchemaInferrer::infer() consumes; a future parallel driver fans - * SchemaInferrer::sniff() over the inner iterables and merges the ColumnTypes partials. - * The unit itself is never named as a type - the implementer decides what one inner iterable is. + * The outer iterable is exactly what SchemaInferrer::infer() consumes; a unit that implements + * SniffsColumnTypes folds itself into its partial instead of being iterated row by row. * * @return iterable> */ diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php new file mode 100644 index 0000000000..97b55c0a3d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php @@ -0,0 +1,28 @@ + + */ +interface SniffsColumnTypes extends Traversable +{ + /** + * Must equal SchemaInferrer::sniff($names, , $rowBudget). + * + * @param list $names - columns known before any row + * @param int<0, max>|-1 $rowBudget - rows to observe, 0 for none, -1 for all of them + */ + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes; +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php new file mode 100644 index 0000000000..f63085dfce --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php @@ -0,0 +1,51 @@ + + */ +final class SpySniffingSample implements IteratorAggregate, SniffsColumnTypes +{ + /** + * @var list, int}> + */ + public array $sniffed = []; + + public function __construct( + private readonly ColumnTypes $fold, + ) {} + + /** + * @return ArrayIterator + */ + public function getIterator(): ArrayIterator + { + throw new RuntimeException('a sample that sniffs itself is never iterated'); + } + + public function sniffColumnTypes( + array $names, + int $rowBudget, + SchemaInference $inference, + TypeNarrower $typer, + ): ColumnTypes { + $this->sniffed[] = [$names, $rowBudget]; + + return $this->fold; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php index 9b3d51eaab..e8e25a9228 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnTypesMother.php @@ -7,6 +7,7 @@ use Flow\ETL\Schema\Inference\ColumnTypes; use Flow\ETL\Schema\Inference\InferredTypes; use Flow\ETL\Schema\Inference\TypeFloor; +use Flow\Types\Type; use Flow\Types\Type\Logical\InstanceOfTypeNarrower; use Flow\Types\Type\Native\String\StringTypeNarrower; @@ -24,6 +25,16 @@ public static function floor(): TypeFloor return new TypeFloor(InferredTypes::default()); } + /** + * A fold computed elsewhere - a native reader, a subprocess. + * + * @param array> $types + */ + public static function fromColumnTypes(array $types, int $rows): ColumnTypes + { + return ColumnTypes::fromColumnTypes($types, $rows, new StringTypeNarrower(InferredTypes::default()->toArray())); + } + /** * @param list $names */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php index 52c598934b..1fe18e5146 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php @@ -274,6 +274,51 @@ public function test_an_optional_non_string_column_is_not_saturated(): void ); } + public function test_from_column_types_reproduces_an_observed_fold(): void + { + $observed = ColumnTypesMother::fromStrings(['id', 'name']); + $observed->observe(new RawRowValues(['id' => '1', 'name' => 'a'])); + $observed->observe(new RawRowValues(['id' => '2', 'name' => null])); + + $computed = ColumnTypesMother::fromColumnTypes([ + 'id' => type_optional(type_integer()), + 'name' => type_optional(type_string()), + ], 2); + + static::assertEquals( + $observed->schema(ColumnTypesMother::floor()), + $computed->schema(ColumnTypesMother::floor()), + ); + static::assertSame($observed->rows(), $computed->rows()); + } + + public function test_from_column_types_merges_with_an_observed_fold(): void + { + $left = ColumnTypesMother::fromStrings(['a']); + $left->observe(new RawRowValues(['a' => '1'])); + $right = ColumnTypesMother::fromStrings(['a']); + $right->observe(new RawRowValues(['a' => '1.5', 'b' => 'x'])); + + static::assertEquals( + $left->merge($right, true)->schema(ColumnTypesMother::floor()), + $left->merge(ColumnTypesMother::fromColumnTypes([ + 'a' => type_optional(type_float()), + 'b' => type_string(), + ], 1), true)->schema(ColumnTypesMother::floor()), + ); + } + + public function test_from_column_types_carries_the_row_count_into_merge(): void + { + $observed = ColumnTypesMother::fromStrings(); + $observed->observe(new RawRowValues(['a' => '1'])); + + static::assertSame( + 8, + $observed->merge(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 7), true)->rows(), + ); + } + public function test_merge_is_associative_over_three_partials(): void { $left = ColumnTypesMother::fromStrings(['a']); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php index 63b6d88b3b..b6b1a732b1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Schema\Inference\SchemaInferrer; use Flow\ETL\Tests\Double\FakeSchemaSampler; use Flow\ETL\Tests\Double\RecordingSources; +use Flow\ETL\Tests\Double\SpySniffingSample; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\ColumnTypesMother; use Flow\Types\Type\Native\String\StringTypeNarrower; @@ -20,6 +21,7 @@ use function array_slice; use function Flow\ETL\DSL\definition_from_type; use function Flow\Types\DSL\type_date; +use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; @@ -78,6 +80,43 @@ public static function threeSourcesOfTenRows(): array return $sources; } + public function test_a_sample_that_sniffs_itself_is_folded_with_the_budget_left(): void + { + $first = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 4)); + $second = new SpySniffingSample(ColumnTypesMother::fromColumnTypes([ + 'a' => type_float(), + 'b' => type_string(), + ], 3)); + + $schema = (new SchemaInferrer( + new SchemaInference(sampleSize: 10, unionByName: true), + new StringTypeNarrower(), + ))->infer(['a'], [$first, $second]); + + static::assertSame([[['a'], 10]], $first->sniffed); + static::assertSame([[['a'], 6]], $second->sniffed); + static::assertEquals( + new Schema( + definition_from_type('a', type_float(), nullable: true), + definition_from_type('b', type_string(), nullable: true), + ), + $schema, + ); + } + + public function test_a_sample_that_sniffs_itself_is_not_asked_once_the_budget_is_spent(): void + { + $first = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 10)); + $second = new SpySniffingSample(ColumnTypesMother::fromColumnTypes(['a' => type_integer()], 10)); + + (new SchemaInferrer(new SchemaInference(sampleSize: 10), new StringTypeNarrower()))->infer(['a'], [ + $first, + $second, + ]); + + static::assertSame([], $second->sniffed); + } + public function test_a_column_that_is_free_text_in_the_first_source_and_numeric_in_the_second_widens_to_string(): void { $fixture = [ diff --git a/src/extension/flow-php-ext/Cargo.lock b/src/extension/flow-php-ext/Cargo.lock index 0d97510639..2dd3b5705b 100644 --- a/src/extension/flow-php-ext/Cargo.lock +++ b/src/extension/flow-php-ext/Cargo.lock @@ -446,6 +446,7 @@ name = "flow_php" version = "0.0.0" dependencies = [ "ext-php-rs", + "memchr", "serde", "serde_json", ] diff --git a/src/extension/flow-php-ext/Cargo.toml b/src/extension/flow-php-ext/Cargo.toml index b7ed1188bb..0eb36b00b0 100644 --- a/src/extension/flow-php-ext/Cargo.toml +++ b/src/extension/flow-php-ext/Cargo.toml @@ -8,5 +8,6 @@ crate-type = ["cdylib"] [dependencies] ext-php-rs = "0.15" +memchr = "2" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["raw_value"] } diff --git a/src/extension/flow-php-ext/composer.json b/src/extension/flow-php-ext/composer.json index e78a7ed00f..c21e562074 100644 --- a/src/extension/flow-php-ext/composer.json +++ b/src/extension/flow-php-ext/composer.json @@ -9,6 +9,7 @@ }, "autoload": { "psr-4": { + "Flow\\ETL\\Adapter\\CSV\\": "php/Flow/ETL/Adapter/CSV/", "Flow\\ETL\\Row\\": "php/Flow/ETL/Row/", "Flow\\Floe\\": "php/Flow/Floe/" } diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php new file mode 100644 index 0000000000..aac509ac5c --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php @@ -0,0 +1,66 @@ +|-1 $limit + * + * @return int<0, max> + */ + public function fold(RustColumnFoldNative $fold, int $limit): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return list + */ + public function headers(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @param int<1, max> $batchSize + * + * @return list + */ + public function next(int $batchSize): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php new file mode 100644 index 0000000000..57e98214ee --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php @@ -0,0 +1,46 @@ + $names + * @param list $candidates - the candidate types' toString() + */ + public function __construct(array $names, array $candidates) + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function narrowOne(string $value): string + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return int<0, max> + */ + public function rows(): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return array + */ + public function types(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/src/csv/fold.rs b/src/extension/flow-php-ext/src/csv/fold.rs new file mode 100644 index 0000000000..66aad50fb5 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/fold.rs @@ -0,0 +1,446 @@ +//! Schema inference over CSV cells, exactly as `ColumnTypes::observe()` + `StringTypeNarrower::narrow()` + +//! `TypeWidener::widen()` fold them, for the closed set of types a CSV cell can narrow to. `json_validate()`, +//! `date_parse()` and `new DateTimeZone('+HH:MM')` are called back into PHP, each behind a Rust pre-filter that is a +//! necessary condition of the PHP predicate, so parity on those rungs holds by construction. + +use std::collections::HashMap; + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::ctx::{call_handle, zval_str, Ctx}; +use crate::csv::php_trim; +use crate::csv::tokenizer::is_space; +use crate::exception::ext_exception; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Leaf { + Null, + String, + Json, + Uuid, + Float, + Integer, + DateTime, + Date, + Boolean, + TimeZone, +} + +impl Leaf { + pub fn code(self) -> &'static str { + match self { + Leaf::Null => "null", + Leaf::String => "string", + Leaf::Json => "json", + Leaf::Uuid => "uuid", + Leaf::Float => "float", + Leaf::Integer => "integer", + Leaf::DateTime => "datetime", + Leaf::Date => "date", + Leaf::Boolean => "boolean", + Leaf::TimeZone => "timezone", + } + } +} + +/// A leaf, optionally nullable; `Null` itself is never optional. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Kind { + leaf: Leaf, + optional: bool, +} + +impl Kind { + fn of(leaf: Leaf) -> Self { + Self { leaf, optional: false } + } + + pub fn code(self) -> String { + if self.optional { + format!("?{}", self.leaf.code()) + } else { + self.leaf.code().to_string() + } + } + + fn is_string(self) -> bool { + self.leaf == Leaf::String + } +} + +/// `TypeWidener::widen()` over the CSV leaves; pinned against it pair by pair (`048_csv_widen_parity.phpt`). +fn widen(left: Kind, right: Kind) -> Kind { + if left == right { + return left; + } + + let optional = left.optional || right.optional || left.leaf == Leaf::Null || right.leaf == Leaf::Null; + + let leaf = match (left.leaf, right.leaf) { + (Leaf::Null, leaf) | (leaf, Leaf::Null) => leaf, + (a, b) if a == b => a, + (Leaf::Integer | Leaf::Float, Leaf::Integer | Leaf::Float) => Leaf::Float, + (Leaf::Date | Leaf::DateTime, Leaf::Date | Leaf::DateTime) => Leaf::DateTime, + _ => Leaf::String, + }; + + Kind { leaf, optional } +} + +/// The rungs `StringTypeNarrower::emitsClass()` lets run, by the candidate types' `toString()`. +struct Candidates { + json: bool, + uuid: bool, + float: bool, + integer: bool, + datetime: bool, + date: bool, + boolean: bool, + timezone: bool, +} + +impl Candidates { + /// The fold has no HTML or XML rung, so a candidate set that allows either cannot be folded natively. + fn from_codes(codes: &[Vec]) -> Result { + let has = |code: &[u8]| codes.iter().any(|candidate| candidate == code); + + if has(b"html") || has(b"xml") { + return Err(ext_exception("flow_php cannot fold html or xml candidates natively")); + } + + Ok(Self { + json: has(b"json"), + uuid: has(b"uuid"), + float: has(b"float"), + integer: has(b"integer"), + datetime: has(b"datetime"), + date: has(b"date"), + boolean: has(b"boolean"), + timezone: has(b"timezone"), + }) + } +} + +/// `StringTypeNarrower::narrow()` for a string cell, without the HTML and XML rungs. +pub struct Narrower { + candidates: Candidates, + ctx: Ctx, +} + +impl Narrower { + fn new(candidates: Candidates) -> Result { + Ok(Self { + candidates, + ctx: Ctx::new()?, + }) + } + + pub fn narrow(&mut self, value: &[u8]) -> Result { + let value = php_trim(value); + + if value.is_empty() { + return Ok(Leaf::String); + } + + if is_null(value) { + return Ok(Leaf::Null); + } + + if self.candidates.json && is_json_shaped(value) && self.is_json(value)? { + return Ok(Leaf::Json); + } + + if self.candidates.uuid && is_uuid(value) { + return Ok(Leaf::Uuid); + } + + if self.candidates.float && is_float(value) { + return Ok(Leaf::Float); + } + + if self.candidates.integer && is_integer(value) { + return Ok(Leaf::Integer); + } + + if (self.candidates.datetime || self.candidates.date) && has_explicit_day(value) { + match self.temporal(value)? { + Some(Leaf::DateTime) if self.candidates.datetime => return Ok(Leaf::DateTime), + Some(Leaf::Date) if self.candidates.date => return Ok(Leaf::Date), + _ => {} + } + } + + if self.candidates.boolean && (value.eq_ignore_ascii_case(b"true") || value.eq_ignore_ascii_case(b"false")) { + return Ok(Leaf::Boolean); + } + + if self.candidates.timezone && self.is_timezone(value)? { + return Ok(Leaf::TimeZone); + } + + Ok(Leaf::String) + } + + fn is_json(&mut self, value: &[u8]) -> Result { + let valid = call_handle(self.ctx.json_validate()?, None, &mut [zval_str(value)], "validate a JSON cell")?; + + Ok(valid.bool().unwrap_or(false)) + } + + /// `StringTemporalParts::from()` past its `hasExplicitDay()` gate: `Date`, `DateTime`, or `None` when the value + /// is not a calendar date. + fn temporal(&mut self, value: &[u8]) -> Result, PhpException> { + let parts_zv = call_handle(self.ctx.date_parse()?, None, &mut [zval_str(value)], "parse a temporal cell")?; + let parts = parts_zv + .array() + .ok_or_else(|| ext_exception("flow_php expected date_parse() to return an array"))?; + + let long = |key: &str| parts.get(key).and_then(Zval::long); + let present = |key: &str| parts.get(key).is_some_and(|zv| !zv.is_false()); + + if long("error_count").unwrap_or(0) > 0 { + return Ok(None); + } + + let (Some(year), Some(month), Some(day)) = (long("year"), long("month"), long("day")) else { + return Ok(None); + }; + + if !checkdate(month, day, year) { + return Ok(None); + } + + let mut time = present("hour") || present("minute") || present("second") || present("fraction"); + + if !time { + if let Some(relative) = parts.get("relative").and_then(Zval::array) { + time = ["hour", "minute", "second"] + .iter() + .any(|key| relative.get(*key).is_some_and(|zv| !is_zero(zv))); + } + } + + Ok(Some(if time { Leaf::DateTime } else { Leaf::Date })) + } + + fn is_timezone(&mut self, value: &[u8]) -> Result { + if self.ctx.timezone_identifiers()?.contains(value) { + return Ok(true); + } + + Ok(is_offset(value) && self.ctx.timezone_accepts(value)?) + } +} + +fn is_zero(zv: &Zval) -> bool { + zv.long() == Some(0) +} + +/// `in_array(mb_strtolower($value), ['null', 'nil'], true)`: no non-ASCII codepoint lowercases into these letters. +fn is_null(value: &[u8]) -> bool { + value.is_ascii() && (value.eq_ignore_ascii_case(b"null") || value.eq_ignore_ascii_case(b"nil")) +} + +/// `Json::isValid()`'s shape gate before `json_validate()`. +fn is_json_shaped(value: &[u8]) -> bool { + matches!((value.first(), value.last()), (Some(b'{'), Some(b'}')) | (Some(b'['), Some(b']'))) +} + +/// `Uuid::isValid()`: 36 bytes, `/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/`. +fn is_uuid(value: &[u8]) -> bool { + value.len() == 36 + && value.iter().enumerate().all(|(index, byte)| match index { + 8 | 13 | 18 | 23 => *byte == b'-', + _ => byte.is_ascii_digit() || (b'a'..=b'f').contains(byte), + }) +} + +/// PHP 8 `is_numeric()` on a string: optional leading and trailing whitespace, optional sign, +/// `digits[.digits] | .digits | digits.`, optional exponent. No hex, no `INF`/`NAN`. +fn is_numeric(value: &[u8]) -> bool { + let mut position = value.iter().take_while(|byte| is_space(**byte)).count(); + let end = value.len() - value[position..].iter().rev().take_while(|byte| is_space(**byte)).count(); + + if matches!(value.get(position), Some(b'+' | b'-')) { + position += 1; + } + + let digits = |from: usize| value[from..end].iter().take_while(|byte| byte.is_ascii_digit()).count(); + + let integral = digits(position); + position += integral; + + let mut fractional = 0; + + if value.get(position) == Some(&b'.') && position < end { + fractional = digits(position + 1); + position += 1 + fractional; + } + + if integral == 0 && fractional == 0 { + return false; + } + + if position < end && matches!(value[position], b'e' | b'E') { + let mut exponent = position + 1; + + if exponent < end && matches!(value[exponent], b'+' | b'-') { + exponent += 1; + } + + let exponent_digits = digits(exponent); + + if exponent_digits == 0 { + return false; + } + + position = exponent + exponent_digits; + } + + position == end +} + +/// `StringTypeNarrower::isFloat()`. +fn is_float(value: &[u8]) -> bool { + is_numeric(value) && value.iter().any(|byte| matches!(byte, b'.' | b'e' | b'E')) +} + +/// `is_numeric($value) && (string) (int) $value === $value`: the canonical decimal form of an int64. +fn is_integer(value: &[u8]) -> bool { + std::str::from_utf8(value) + .ok() + .and_then(|string| string.parse::().ok()) + .is_some_and(|parsed| parsed.to_string().as_bytes() == value) +} + +/// `StringTemporalParts::hasExplicitDay()`. +fn has_explicit_day(value: &[u8]) -> bool { + let groups = value + .iter() + .enumerate() + .filter(|(index, byte)| byte.is_ascii_digit() && (*index == 0 || !value[index - 1].is_ascii_digit())) + .count(); + + if groups >= 3 { + return true; + } + + if groups == 1 && value.len() == 8 && value.iter().all(u8::is_ascii_digit) { + return true; + } + + groups >= 2 && value.windows(3).any(|window| window.iter().all(u8::is_ascii_alphabetic)) +} + +/// `StringTypeNarrower::isTimeZone()`'s `/^[+-]\d{2}:\d{2}$/` on a trimmed value. +fn is_offset(value: &[u8]) -> bool { + matches!(value, [b'+' | b'-', h1, h2, b':', m1, m2] if [h1, h2, m1, m2].iter().all(|byte| byte.is_ascii_digit())) +} + +/// `checkdate()`. +fn checkdate(month: i64, day: i64, year: i64) -> bool { + if !(1..=32767).contains(&year) || !(1..=12).contains(&month) || day < 1 { + return false; + } + + let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + let days = match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + + day <= days +} + +struct Column { + name: Vec, + kind: Kind, +} + +/// `ColumnTypes::observe()` over whole rows: names seeded `null` (then widened), names met only in rows start at +/// their first observed type; a `string` / `?string` column skips non-null cells. +pub struct Fold { + pub narrower: Narrower, + columns: Vec, + index: HashMap, usize>, + rows: u64, +} + +impl Fold { + pub fn new(names: Vec>, candidates: &[Vec]) -> Result { + let mut fold = Self { + narrower: Narrower::new(Candidates::from_codes(candidates)?)?, + columns: Vec::new(), + index: HashMap::new(), + rows: 0, + }; + + for name in names { + if !fold.index.contains_key(&name) { + fold.index.insert(name.clone(), fold.columns.len()); + fold.columns.push(Column { + name, + kind: Kind::of(Leaf::Null), + }); + } + } + + Ok(fold) + } + + /// One cell of the current row; `None` is a null cell. `position` caches the cell's column across rows. + pub fn observe(&mut self, name: &[u8], position: &mut Option, value: Option<&[u8]>) -> Result<(), PhpException> { + if position.is_none() { + *position = self.index.get(name).copied(); + } + + if value.is_some() && position.is_some_and(|position| self.columns[position].kind.is_string()) { + return Ok(()); + } + + let observed = Kind::of(match value { + None => Leaf::Null, + Some(value) if php_trim(value).len() != value.len() => Leaf::String, + Some(value) => match self.narrower.narrow(value)? { + Leaf::Null => Leaf::String, + leaf => leaf, + }, + }); + + match *position { + Some(position) => self.columns[position].kind = widen(self.columns[position].kind, observed), + None => { + self.index.insert(name.to_vec(), self.columns.len()); + *position = Some(self.columns.len()); + self.columns.push(Column { + name: name.to_vec(), + kind: observed, + }); + } + } + + Ok(()) + } + + pub fn end_row(&mut self) { + self.rows += 1; + } + + pub fn rows(&self) -> u64 { + self.rows + } + + /// Column name => type code, first-seen order. + pub fn types(&self) -> impl Iterator { + self.columns.iter().map(|column| (column.name.as_slice(), column.kind.code())) + } +} + +pub fn string_values(list: &ZendHashTable) -> Vec> { + list.values() + .filter_map(|zv| zv.zend_str().map(|value| value.as_bytes().to_vec())) + .collect() +} diff --git a/src/extension/flow-php-ext/src/csv/mod.rs b/src/extension/flow-php-ext/src/csv/mod.rs new file mode 100644 index 0000000000..979be674b8 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/mod.rs @@ -0,0 +1,263 @@ +//! Native CSV reading: records and fields from `tokenizer`, rows shaped exactly like +//! `CSVEncoder::decode()` + `CSVRowNormalizer::normalize()` shape them. + +pub mod fold; +pub mod tokenizer; + +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; + +use crate::ctx::{array_key_index, ht_insert_key, null_zval, write_slot, zval_str, HtKey}; +use crate::exception::ext_exception; +use crate::hydrate::RowValuesClass; +use tokenizer::{Dialect, Field, Record, Tokenizer}; + +/// A header name resolved once into the array key `array_combine()` would use for it. +enum HeaderKey { + Index(i64), + Str(ZBox), +} + +struct Header { + name: Vec, + key: HeaderKey, +} + +impl Header { + fn new(name: Vec) -> Self { + let key = match array_key_index(&name) { + Some(index) => HeaderKey::Index(index), + None => HeaderKey::Str(ZendStr::new(&name, false)), + }; + + Self { name, key } + } + + fn key(&self) -> HtKey<'_> { + match &self.key { + HeaderKey::Index(index) => HtKey::Index(*index), + HeaderKey::Str(name) => HtKey::Str(name), + } + } +} + +/// One key of `RawRowValues::$values`: `array_combine()` keeps a duplicated header at its first position with the +/// value of its last one. +struct Cell { + header_index: usize, + field_index: usize, +} + +pub struct CsvReader { + tokenizer: Tokenizer, + with_header: bool, + empty_to_null: bool, + headers: Option>, + cells: Vec, + record: Record, + first_row_pending: bool, +} + +impl CsvReader { + pub fn new( + separator: &[u8], + enclosure: &[u8], + escape: &[u8], + with_header: bool, + empty_to_null: bool, + remove_bom: bool, + ) -> Result { + let [separator] = separator else { + return Err(ext_exception("flow_php CSV separator must be exactly one byte")); + }; + let [enclosure] = enclosure else { + return Err(ext_exception("flow_php CSV enclosure must be exactly one byte")); + }; + let escape = match escape { + [] => None, + [escape] => Some(*escape), + _ => return Err(ext_exception("flow_php CSV escape must be empty or exactly one byte")), + }; + + Ok(Self { + tokenizer: Tokenizer::new( + Dialect { + separator: *separator, + enclosure: *enclosure, + escape, + }, + remove_bom, + ), + with_header, + empty_to_null, + headers: None, + cells: Vec::new(), + record: Record::default(), + first_row_pending: false, + }) + } + + pub fn feed(&mut self, chunk: &[u8]) { + self.tokenizer.feed(chunk); + } + + pub fn finish(&mut self) { + self.tokenizer.finish(); + } + + /// The resolved header, `[]` until the first record is available. + pub fn headers(&mut self) -> Vec<&[u8]> { + self.resolve_headers(); + + self.headers + .as_ref() + .map(|headers| headers.iter().map(|header| header.name.as_slice()).collect()) + .unwrap_or_default() + } + + /// Up to `batch_size` `RawRowValues`; an empty list when no complete record is buffered. + pub fn next(&mut self, batch_size: usize, class: &RowValuesClass) -> Result, PhpException> { + self.resolve_headers(); + + let mut batch = ZendHashTable::new(); + + let Some(headers) = self.headers.as_ref() else { + return Ok(batch); + }; + + while batch.len() < batch_size { + if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record) { + break; + } + + batch + .push(row_values(headers, &self.record, self.empty_to_null, class)) + .map_err(|e| ext_exception(format!("flow_php failed to collect CSV row values: {e:?}")))?; + } + + Ok(batch) + } + + /// Folds up to `limit` rows (all when `None`) into `fold`; returns how many it folded - fewer only when no + /// further complete record is buffered. + pub fn fold(&mut self, fold: &mut fold::Fold, limit: Option) -> Result { + self.resolve_headers(); + + let Some(headers) = self.headers.as_ref() else { + return Ok(0); + }; + + let mut folded = 0; + let mut positions = vec![None; self.cells.len()]; + + while limit.is_none_or(|limit| folded < limit) { + if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record) { + break; + } + + for (cell, position) in self.cells.iter().zip(positions.iter_mut()) { + fold.observe( + &headers[cell.header_index].name, + position, + cell_value(&self.record, cell.field_index, self.empty_to_null), + )?; + } + + fold.end_row(); + folded += 1; + } + + Ok(folded) + } + + /// `CSVEncoder::decode()`'s first line: the header when `withHeader`, else `e00, e01, ...` sized by + /// the first record, which then stays pending as the first row. + fn resolve_headers(&mut self) { + if self.headers.is_some() || !self.tokenizer.next(&mut self.record) { + return; + } + + let record = &self.record; + + self.headers = Some(if self.with_header { + (0..record.len()) + .map(|index| { + let trimmed = php_trim(match record.field(index) { + Field::Value(value) => value, + Field::Null | Field::Missing => b"", + }); + + Header::new(if trimmed.is_empty() { + format!("e{index:02}").into_bytes() + } else { + trimmed.to_vec() + }) + }) + .collect() + } else { + self.first_row_pending = true; + + (0..record.len()).map(|index| Header::new(format!("e{index:02}").into_bytes())).collect() + }); + + let headers = self.headers.as_ref().expect("resolved above"); + + for (index, header) in headers.iter().enumerate() { + match self.cells.iter_mut().find(|cell| headers[cell.header_index].name == header.name) { + Some(cell) => cell.field_index = index, + None => self.cells.push(Cell { + header_index: index, + field_index: index, + }), + } + } + } +} + +/// The first row is pending when auto headers were sized by it; otherwise the next complete record. +fn next_row(first_row_pending: &mut bool, tokenizer: &mut Tokenizer, record: &mut Record) -> bool { + std::mem::take(first_row_pending) || tokenizer.next(record) +} + +/// `CSVRowNormalizer::normalize()` for one cell: padding past the record's end, `emptyToNull`; `None` is null. +fn cell_value(record: &Record, index: usize, empty_to_null: bool) -> Option<&[u8]> { + match record.field(index) { + Field::Value(value) if !(empty_to_null && value.is_empty()) => Some(value), + Field::Value(_) | Field::Null => None, + Field::Missing if empty_to_null => None, + Field::Missing => Some(b""), + } +} + +/// `new RawRowValues(array_combine($headers, $normalizer->normalize($fields, count($headers))))`. +fn row_values(headers: &[Header], record: &Record, empty_to_null: bool, class: &RowValuesClass) -> ZBox { + let mut values = ZendHashTable::with_capacity(headers.len() as u32); + + for (index, header) in headers.iter().enumerate() { + let value = cell_value(record, index, empty_to_null).map_or_else(null_zval, zval_str); + + ht_insert_key(&mut values, &header.key(), value); + } + + let mut row_values = ZendObject::new(class.ce); + + let mut values_zv = Zval::new(); + values_zv.set_hashtable(values); + write_slot(&mut row_values, class.values_slot, values_zv); + + let mut metadata_zv = Zval::new(); + metadata_zv.set_hashtable(ZendHashTable::new()); + write_slot(&mut row_values, class.metadata_slot, metadata_zv); + + row_values +} + +/// PHP's `trim()`: `" \t\n\r\0\x0B"`, nothing else. +pub fn php_trim(bytes: &[u8]) -> &[u8] { + let is_trimmed = |byte: &u8| matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\0' | 0x0B); + let start = bytes.iter().position(|byte| !is_trimmed(byte)).unwrap_or(bytes.len()); + let end = bytes.iter().rposition(|byte| !is_trimmed(byte)).map_or(start, |end| end + 1); + + &bytes[start..end] +} diff --git a/src/extension/flow-php-ext/src/csv/tokenizer.rs b/src/extension/flow-php-ext/src/csv/tokenizer.rs new file mode 100644 index 0000000000..a3af8a56f0 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/tokenizer.rs @@ -0,0 +1,344 @@ +//! CSV records and fields exactly as the PHP path produces them: `CSVLineReader` assembles a record +//! (a `\n` ends it only outside an enclosure, the record is `rtrim($buffer, "\r\n")`, the first one +//! loses its BOM) and `str_getcsv()` splits it - a port of `php_fgetcsv()` (`ext/standard/file.c`). + +use std::ops::Range; + +use memchr::{memchr, memchr2}; + +pub struct Dialect { + pub separator: u8, + pub enclosure: u8, + pub escape: Option, +} + +/// One record's fields as ranges of `bytes`; no field at all is a blank line (`str_getcsv('')` is `[null]`). +#[derive(Default)] +pub struct Record { + bytes: Vec, + fields: Vec>, +} + +pub enum Field<'a> { + Value(&'a [u8]), + Null, + Missing, +} + +impl Record { + pub fn len(&self) -> usize { + self.fields.len().max(1) + } + + pub fn field(&self, index: usize) -> Field<'_> { + match self.fields.get(index) { + Some(range) => Field::Value(&self.bytes[range.clone()]), + None if index == 0 && self.fields.is_empty() => Field::Null, + None => Field::Missing, + } + } +} + +/// Where the record scan stands after the last byte it looked at. +#[derive(Clone, Copy)] +enum Scan { + FieldStart, + Unenclosed, + Enclosed, + Escaped, + ClosingOrDoubled, +} + +const BOMS: [&[u8]; 5] = [ + b"\xEF\xBB\xBF", + b"\xFF\xFE\x00\x00", + b"\x00\x00\xFE\xFF", + b"\xFF\xFE", + b"\xFE\xFF", +]; + +/// Resumable across chunk boundaries: `feed` appends bytes, `next` fills the next complete record or returns +/// `false` while it is not yet closed. A record left open at EOF is emitted as-is (PHP's `unterminated` case). +pub struct Tokenizer { + dialect: Dialect, + remove_bom: bool, + buffer: Vec, + record_start: usize, + scanned: usize, + scan: Scan, + finished: bool, + first_record: bool, +} + +impl Tokenizer { + pub fn new(dialect: Dialect, remove_bom: bool) -> Self { + Self { + dialect, + remove_bom, + buffer: Vec::new(), + record_start: 0, + scanned: 0, + scan: Scan::FieldStart, + finished: false, + first_record: true, + } + } + + pub fn feed(&mut self, chunk: &[u8]) { + if self.record_start > 0 { + self.buffer.drain(..self.record_start); + self.scanned -= self.record_start; + self.record_start = 0; + } + + self.buffer.extend_from_slice(chunk); + } + + pub fn finish(&mut self) { + self.finished = true; + } + + pub fn next(&mut self, record: &mut Record) -> bool { + let Some(range) = self.next_record() else { + return false; + }; + + self.split(range, record); + + true + } + + /// The next complete record as a range of `buffer`, after `rtrim("\r\n")` and the BOM strip. + fn next_record(&mut self) -> Option> { + let mut record = match self.scan_to_record_end() { + Some(newline) => { + let range = self.record_start..newline; + self.record_start = newline + 1; + self.scanned = self.record_start; + self.scan = Scan::FieldStart; + + range + } + None if self.finished && self.record_start < self.buffer.len() => { + let range = self.record_start..self.buffer.len(); + self.record_start = self.buffer.len(); + self.scanned = self.record_start; + self.scan = Scan::FieldStart; + + range + } + None => return None, + }; + + while record.end > record.start && matches!(self.buffer[record.end - 1], b'\r' | b'\n') { + record.end -= 1; + } + + if self.remove_bom && self.first_record { + if let Some(bom) = BOMS.iter().find(|bom| self.buffer[record.clone()].starts_with(bom)) { + record.start += bom.len(); + } + } + + self.first_record = false; + + Some(record) + } + + /// Advances the scan over unseen bytes; returns the position of the `\n` that ends the record. + fn scan_to_record_end(&mut self) -> Option { + let Dialect { separator, enclosure, escape } = self.dialect; + let bytes = &self.buffer; + let mut position = self.scanned; + let mut scan = self.scan; + + let found = loop { + if position >= bytes.len() { + break None; + } + + match scan { + Scan::FieldStart => { + let byte = bytes[position]; + + if byte == b'\n' { + break Some(position); + } + + scan = if byte == separator { + Scan::FieldStart + } else if byte == enclosure { + Scan::Enclosed + } else if is_space(byte) { + Scan::FieldStart + } else { + Scan::Unenclosed + }; + position += 1; + } + Scan::Unenclosed => match memchr2(separator, b'\n', &bytes[position..]) { + Some(offset) if bytes[position + offset] == b'\n' => break Some(position + offset), + Some(offset) => { + position += offset + 1; + scan = Scan::FieldStart; + } + None => position = bytes.len(), + }, + Scan::Enclosed => { + let special = match escape { + Some(escape) if escape != enclosure => memchr2(enclosure, escape, &bytes[position..]), + _ => memchr(enclosure, &bytes[position..]), + }; + + match special { + Some(offset) => { + scan = if bytes[position + offset] == enclosure { + Scan::ClosingOrDoubled + } else { + Scan::Escaped + }; + position += offset + 1; + } + None => position = bytes.len(), + } + } + Scan::Escaped => { + scan = Scan::Enclosed; + position += 1; + } + Scan::ClosingOrDoubled => { + if bytes[position] == enclosure { + scan = Scan::Enclosed; + position += 1; + } else { + scan = Scan::Unenclosed; + } + } + } + }; + + self.scanned = position; + self.scan = scan; + + found + } + + /// `str_getcsv($record, ...)` into `record`. + fn split(&self, range: Range, record: &mut Record) { + let Dialect { separator, enclosure, escape } = self.dialect; + let escape = escape.filter(|escape| *escape != enclosure); + let line = &self.buffer[range]; + // next_record() already stripped every trailing \r and \n, so php_fgetcsv()'s line end is always empty here + let limit = line.len(); + + record.bytes.clear(); + record.fields.clear(); + + if limit == 0 { + return; + } + + let mut position = 0; + + loop { + let field_start = record.bytes.len(); + + if position < limit { + let mut skipped = position; + + while skipped < limit && line[skipped] != separator && is_space(line[skipped]) { + skipped += 1; + } + + if skipped < limit && line[skipped] == enclosure { + position = skipped; + } + } + + let delimiter = if position < limit && line[position] == enclosure { + position += 1; + let mut hunk = position; + + let unterminated = loop { + let Some(offset) = (match escape { + Some(escape) => memchr2(enclosure, escape, &line[position..limit]), + None => memchr(enclosure, &line[position..limit]), + }) else { + break true; + }; + + position += offset; + + if line[position] != enclosure { + position += 2; + + if position > limit { + break true; + } + + continue; + } + + if position + 1 < limit && line[position + 1] == enclosure { + record.bytes.extend_from_slice(&line[hunk..=position]); + position += 2; + hunk = position; + + continue; + } + + record.bytes.extend_from_slice(&line[hunk..position]); + position += 1; + hunk = position; + + break false; + }; + + if unterminated { + record.bytes.extend_from_slice(&line[hunk..limit]); + hunk = limit; + position = limit; + } + + let delimiter = memchr(separator, &line[position..limit]).map(|offset| position + offset); + let end = delimiter.unwrap_or(limit); + + if hunk < end { + record.bytes.extend_from_slice(&line[hunk..end]); + } + + delimiter + } else { + let delimiter = memchr(separator, &line[position..limit]).map(|offset| position + offset); + + record.bytes.extend_from_slice(&line[position..delimiter.unwrap_or(limit)]); + + let trimmed = field_start + trailing_line_end(&record.bytes[field_start..]); + record.bytes.truncate(trimmed); + + delimiter + }; + + record.fields.push(field_start..record.bytes.len()); + + match delimiter { + Some(delimiter) => position = delimiter + 1, + None => break, + } + } + } +} + +/// `isspace()` in the C locale. +pub fn is_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0B | 0x0C | b'\r') +} + +/// `php_fgetcsv_lookup_trailing_spaces()`: the length without one trailing `\r\n`, `\n` or `\r`. +fn trailing_line_end(bytes: &[u8]) -> usize { + match bytes { + [.., b'\r', b'\n'] => bytes.len() - 2, + [.., b'\n'] | [.., b'\r'] => bytes.len() - 1, + _ => bytes.len(), + } +} diff --git a/src/extension/flow-php-ext/src/ctx.rs b/src/extension/flow-php-ext/src/ctx.rs index fe3b3dc8b6..be66349852 100644 --- a/src/extension/flow-php-ext/src/ctx.rs +++ b/src/extension/flow-php-ext/src/ctx.rs @@ -1,7 +1,7 @@ //! PHP-engine plumbing: class-entry/slot lookups, hashtable helpers, calls and //! per-instance caches (timezones, enums, callables). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use ext_php_rs::boxed::ZBox; use ext_php_rs::convert::IntoZvalDyn; @@ -487,6 +487,9 @@ pub struct Ctx { fn_json_encode: Option, fn_json_decode: Option, fn_json_validate: Option, + fn_date_parse: Option, + timezone_identifiers: Option>>, + timezone_construct: Option<&'static Function>, metadata_from_array: Option<&'static Function>, metadata_map_slot: Option, typed_row_values_slots: Option<(u32, u32)>, @@ -531,6 +534,9 @@ impl Ctx { fn_json_encode: None, fn_json_decode: None, fn_json_validate: None, + fn_date_parse: None, + timezone_identifiers: None, + timezone_construct: None, metadata_from_array: None, metadata_map_slot: None, typed_row_values_slots: None, @@ -900,6 +906,58 @@ impl Ctx { Ok(self.fn_json_validate.as_ref().expect("just initialized")) } + pub fn date_parse(&mut self) -> Result<&Function, PhpException> { + if self.fn_date_parse.is_none() { + self.fn_date_parse = Some(function_handle("date_parse")?); + } + + Ok(self.fn_date_parse.as_ref().expect("just initialized")) + } + + /// `DateTimeZone::listIdentifiers()`, fetched once per `Ctx`. + pub fn timezone_identifiers(&mut self) -> Result<&HashSet>, PhpException> { + if self.timezone_identifiers.is_none() { + let list = call_handle( + method_handle_ref("DateTimeZone", "listIdentifiers")?, + None, + &mut [], + "list timezone identifiers", + )?; + let identifiers = list + .array() + .ok_or_else(|| ext_exception("flow_php expected DateTimeZone::listIdentifiers() to return an array"))? + .values() + .filter_map(|zv| zv.zend_str().map(|name| name.as_bytes().to_vec())) + .collect(); + + self.timezone_identifiers = Some(identifiers); + } + + Ok(self.timezone_identifiers.as_ref().expect("just initialized")) + } + + /// Whether `new DateTimeZone($name)` succeeds; the exception it throws otherwise is discarded. + pub fn timezone_accepts(&mut self, name: &[u8]) -> Result { + if self.timezone_ce.is_none() { + self.timezone_ce = Some(find_class("DateTimeZone")?); + } + + let ce = self.timezone_ce.expect("just initialized"); + + if self.timezone_construct.is_none() { + self.timezone_construct = Some(ce_method_ref(ce, "__construct")?); + } + + let timezone = ZendObject::new(ce); + + Ok(call_handle_catching( + self.timezone_construct.expect("just initialized"), + Some(&timezone), + &mut [zval_str(name)], + ) + .is_ok()) + } + /// Static `Flow\ETL\Schema\Metadata::fromArray` handle - builds a Metadata /// value object from a decoded per-value metadata map (the rare path). pub fn metadata_from_array(&mut self) -> Result<&'static Function, PhpException> { diff --git a/src/extension/flow-php-ext/src/lib.rs b/src/extension/flow-php-ext/src/lib.rs index 522c6459eb..ea135e5060 100644 --- a/src/extension/flow-php-ext/src/lib.rs +++ b/src/extension/flow-php-ext/src/lib.rs @@ -1,4 +1,5 @@ mod cast; +mod csv; mod ctx; mod encode; mod exception; @@ -15,7 +16,7 @@ use ext_php_rs::types::{ZendHashTable, Zval}; use ext_php_rs::zend::ModuleEntry; use ext_php_rs::{info_table_end, info_table_row, info_table_start}; -use crate::ctx::{zval_str, Ctx}; +use crate::ctx::{ht_insert, zval_str, Ctx}; use crate::encode::{build_encode_plan, encode_typed_row, expect_object, ht_for_each, read_slot, EncodePlan}; use crate::exception::ext_exception; use crate::format::Reader; @@ -247,6 +248,120 @@ impl RustRowHydratorNative { } } +/// Native counterpart of `CSVLineReader` + `CSVEncoder::decode()`: bytes in, `RawRowValues` out, +/// byte-identical to the PHP path. Resumable - `feed` any chunk size, `finish` at EOF. +#[php_class] +#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustCSVReaderNative")] +pub struct RustCSVReaderNative { + reader: csv::CsvReader, + row_values_class: hydrate::RowValuesClass, +} + +#[php_impl] +impl RustCSVReaderNative { + pub fn __construct( + separator: BinarySlice, + enclosure: BinarySlice, + escape: BinarySlice, + with_header: bool, + empty_to_null: bool, + remove_bom: bool, + ) -> PhpResult { + Ok(Self { + reader: csv::CsvReader::new(&separator, &enclosure, &escape, with_header, empty_to_null, remove_bom)?, + row_values_class: hydrate::RowValuesClass::resolve()?, + }) + } + + pub fn feed(&mut self, chunk: BinarySlice) { + self.reader.feed(&chunk); + } + + pub fn finish(&mut self) { + self.reader.finish(); + } + + pub fn headers(&mut self) -> PhpResult { + let headers = self.reader.headers(); + let mut list = ZendHashTable::with_capacity(headers.len() as u32); + + for header in headers { + list.push(zval_str(header)) + .map_err(|e| ext_exception(format!("flow_php failed to collect a CSV header: {e:?}")))?; + } + + let mut zv = Zval::new(); + zv.set_hashtable(list); + + Ok(zv) + } + + /// Folds up to `limit` buffered rows (-1: all) into `fold`; returns how many it folded. + pub fn fold(&mut self, fold: &mut RustColumnFoldNative, limit: i64) -> PhpResult { + let limit = match limit { + -1 => None, + limit => Some( + u64::try_from(limit).map_err(|_| ext_exception("flow_php CSV fold limit must be -1 or at least 0"))?, + ), + }; + + Ok(self.reader.fold(&mut fold.fold, limit)? as i64) + } + + pub fn next(&mut self, batch_size: i64) -> PhpResult { + let batch_size = usize::try_from(batch_size) + .ok() + .filter(|size| *size > 0) + .ok_or_else(|| ext_exception("flow_php CSV batch size must be greater than 0"))?; + + let mut zv = Zval::new(); + zv.set_hashtable(self.reader.next(batch_size, &self.row_values_class)?); + + Ok(zv) + } +} + +/// `ColumnTypes::observe()` over native CSV rows: a column fold seeded with the header names and gated by the +/// candidate types' `toString()` codes. HTML and XML candidates are never passed - the PHP side folds those itself. +#[php_class] +#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustColumnFoldNative")] +pub struct RustColumnFoldNative { + fold: csv::fold::Fold, +} + +#[php_impl] +impl RustColumnFoldNative { + pub fn __construct(names: &ZendHashTable, candidates: &ZendHashTable) -> PhpResult { + Ok(Self { + fold: csv::fold::Fold::new(csv::fold::string_values(names), &csv::fold::string_values(candidates))?, + }) + } + + /// `StringTypeNarrower::narrow($value)->toString()` for the fold's candidates. + #[php(name = "narrowOne")] + pub fn narrow_one(&mut self, value: BinarySlice) -> PhpResult { + Ok(self.fold.narrower.narrow(&value)?.code().to_string()) + } + + /// Column name => type `toString()`, first-seen order. + pub fn types(&self) -> Zval { + let mut types = ZendHashTable::new(); + + for (name, code) in self.fold.types() { + ht_insert(&mut types, name, zval_str(code.as_bytes())); + } + + let mut zv = Zval::new(); + zv.set_hashtable(types); + + zv + } + + pub fn rows(&self) -> i64 { + self.fold.rows() as i64 + } +} + #[php_module] #[php(startup = "module_startup")] pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { @@ -255,4 +370,6 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { .info_function(php_module_info) .class::() .class::() + .class::() + .class::() } diff --git a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt new file mode 100644 index 0000000000..a12bdbdb8a --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt @@ -0,0 +1,44 @@ +--TEST-- +native CSV fields match str_getcsv() byte for byte on every tokenizer edge case +--SKIPIF-- + +--FILE-- + '612c622c63', + 'quoted' => '2261222c226222', + 'backslash_quote' => '22615c2262222c63', + 'doubled_quote' => '2261222262222c63', + 'space_before_enclosure' => '2261222c2020226222', + 'junk_after_close' => '612c226222782c63', + 'unterminated' => '22756e7465726d696e617465642c61', + 'blank' => '', + 'trailing_sep' => '612c622c', +]; + +foreach ($cases as $case => $hex) { + $input = hex2bin($hex); + $reader = new RustCSVReaderNative(',', '"', '\\', false, false, false); + $reader->feed($input . "\n"); + $reader->finish(); + + $fields = array_values($reader->next(10)[0]->values); + $expected = str_getcsv($input, ',', '"', '\\'); + + echo $case, ': ', $fields === $expected ? 'identical' : 'FAIL ' . json_encode(array_map(static fn(?string $f): ?string => $f === null ? null : bin2hex($f), $fields)), "\n"; +} +?> +--EXPECT-- +plain: identical +quoted: identical +backslash_quote: identical +doubled_quote: identical +space_before_enclosure: identical +junk_after_close: identical +unterminated: identical +blank: identical +trailing_sep: identical diff --git a/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt b/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt new file mode 100644 index 0000000000..1e5e8655f8 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/041_csv_dialect_options.phpt @@ -0,0 +1,73 @@ +--TEST-- +native CSV reading matches the PHP path for every dialect and read option +--SKIPIF-- + +--FILE-- + +--EXPECT-- +";" ' "\\" 111: identical +";" ' "\\" 011: identical +";" ' "\\" 101: identical +";" ' "\\" 110: identical +";" ' "" 111: identical +";" ' "" 011: identical +";" ' "" 101: identical +";" ' "" 110: identical +";" " "\\" 111: identical +";" " "\\" 011: identical +";" " "\\" 101: identical +";" " "\\" 110: identical +";" " "" 111: identical +";" " "" 011: identical +";" " "" 101: identical +";" " "" 110: identical +"|" ' "\\" 111: identical +"|" ' "\\" 011: identical +"|" ' "\\" 101: identical +"|" ' "\\" 110: identical +"|" ' "" 111: identical +"|" ' "" 011: identical +"|" ' "" 101: identical +"|" ' "" 110: identical +"|" " "\\" 111: identical +"|" " "\\" 011: identical +"|" " "\\" 101: identical +"|" " "\\" 110: identical +"|" " "" 111: identical +"|" " "" 011: identical +"|" " "" 101: identical +"|" " "" 110: identical +"\t" ' "\\" 111: identical +"\t" ' "\\" 011: identical +"\t" ' "\\" 101: identical +"\t" ' "\\" 110: identical +"\t" ' "" 111: identical +"\t" ' "" 011: identical +"\t" ' "" 101: identical +"\t" ' "" 110: identical +"\t" " "\\" 111: identical +"\t" " "\\" 011: identical +"\t" " "\\" 101: identical +"\t" " "\\" 110: identical +"\t" " "" 111: identical +"\t" " "" 011: identical +"\t" " "" 101: identical +"\t" " "" 110: identical diff --git a/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt b/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt new file mode 100644 index 0000000000..d1ac835ed3 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/042_csv_multiline_records.phpt @@ -0,0 +1,38 @@ +--TEST-- +native CSV records span lines exactly like CSVLineReader: quoted LF and CRLF, open at EOF, blank lines, BOM before a quote +--SKIPIF-- + +--FILE-- + "a,b\n\"x\ny\",1\n2,3\n", + 'quoted CRLF' => "a,b\r\n\"x\r\ny\",1\r\n2,3\r\n", + 'escaped quote then LF' => "a,b\n\"x\\\"\ny\",1\n", + 'doubled quote then LF' => "a,b\n\"x\"\"\ny\",1\n", + 'bare quote in unenclosed field' => "a,b\nx\"y,1\n\"p\",2\n", + 'open at EOF' => "a,b\n1,\"never closed\nstill open\n", + 'open at EOF without LF' => "a,b\n1,\"never closed", + 'blank lines' => "a,b\n\n1,2\n\n\n", + 'extra CRs' => "a,b\r\r\n1,2\r\r\r\n", + 'BOM then quote' => "\xEF\xBB\xBF\"a\",b\n1,2\n", + 'BOM then open quote' => "\xEF\xBB\xBF\"a\nb\",c\n1,2\n", +]; + +foreach ($cases as $case => $raw) { + assert_csv_identical($case, csv_php_rows($raw, ',', '"', '\\'), csv_native_rows($raw, ',', '"', '\\')); +} +?> +--EXPECT-- +quoted LF: identical +quoted CRLF: identical +escaped quote then LF: identical +doubled quote then LF: identical +bare quote in unenclosed field: identical +open at EOF: identical +open at EOF without LF: identical +blank lines: identical +extra CRs: identical +BOM then quote: identical +BOM then open quote: identical diff --git a/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt b/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt new file mode 100644 index 0000000000..9064f10c3f --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/043_csv_chunk_boundaries.phpt @@ -0,0 +1,24 @@ +--TEST-- +native CSV rows do not depend on where chunk boundaries fall +--SKIPIF-- + +--FILE-- + +--EXPECT-- +whole input vs PHP path: identical +chunk 1: identical +chunk 2: identical +chunk 3: identical +chunk 7: identical +chunk 4096: identical diff --git a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt new file mode 100644 index 0000000000..c447a601d0 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt @@ -0,0 +1,50 @@ +--TEST-- +native CSV rows pad and truncate to the header like CSVRowNormalizer, under both emptyToNull settings +--SKIPIF-- + +--FILE-- + +--EXPECT-- +emptyToNull=1 withHeader=1: identical +emptyToNull=1 withHeader=0: identical +emptyToNull=0 withHeader=1: identical +emptyToNull=0 withHeader=0: identical +array(5) { + [0]=> + string(2) "id" + [1]=> + string(4) "name" + [2]=> + string(3) "e02" + [3]=> + string(1) "5" + [4]=> + string(2) "id" +} +array(4) { + ["id"]=> + NULL + ["name"]=> + NULL + ["e02"]=> + NULL + [5]=> + NULL +} diff --git a/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt new file mode 100644 index 0000000000..b2d946369e --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt @@ -0,0 +1,52 @@ +--TEST-- +repeated native CSV reading does not leak memory +--SKIPIF-- + +--FILE-- +feed($chunk); + $reader->next(2); + } + + $reader->finish(); + $reader->headers(); + + while ($reader->next(2) !== []) { + } + + $unfinished = new RustCSVReaderNative(';', "'", '', false, false, false); + $unfinished->feed("a;'open\n"); + $unfinished->next(10); + + try { + new RustCSVReaderNative(',,', '"', '\\', true, true, true); + throw new LogicException('an invalid separator was accepted'); + } catch (Flow\Floe\Exception\ExtensionException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 10000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt new file mode 100644 index 0000000000..33c1460008 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt @@ -0,0 +1,21 @@ +--TEST-- +native narrowing matches StringTypeNarrower::narrow() on its fixtures, the byte-level traps and a fuzz corpus +--SKIPIF-- + +--FILE-- += 80); +var_dump(count($corpus) >= 300); +assert_narrow_parity('default candidates', InferredTypes::default()->toArray(), $corpus); +?> +--EXPECT-- +bool(true) +bool(true) +default candidates: identical diff --git a/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt b/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt new file mode 100644 index 0000000000..b34c9502e7 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/047_csv_narrow_candidate_gating.phpt @@ -0,0 +1,42 @@ +--TEST-- +native narrowing skips exactly the rungs a restricted candidate set leaves out +--SKIPIF-- + +--FILE-- + new InferredTypes(type_string()), + 'integer' => new InferredTypes(type_integer()), + 'float integer' => new InferredTypes(type_float(), type_integer()), + 'date' => new InferredTypes(type_date()), + 'datetime' => new InferredTypes(type_datetime()), + 'boolean timezone' => new InferredTypes(type_boolean(), type_time_zone()), + 'json uuid' => new InferredTypes(type_json(), type_uuid()), +] as $label => $candidates) { + assert_narrow_parity($label, $candidates->toArray(), $corpus); +} +?> +--EXPECT-- +all strings: identical +integer: identical +float integer: identical +date: identical +datetime: identical +boolean timezone: identical +json uuid: identical diff --git a/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt new file mode 100644 index 0000000000..972de86d42 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt @@ -0,0 +1,71 @@ +--TEST-- +the native fold widens every pair of CSV leaf types exactly like TypeWidener +--SKIPIF-- + +--FILE-- + [null, type_null()], + 'string' => ['x', type_string()], + 'json' => ['{"a":1}', type_json()], + 'uuid' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479', type_uuid()], + 'float' => ['1.5', type_float()], + 'integer' => ['1', type_integer()], + 'datetime' => ['2024-01-01 10:00:00', type_datetime()], + 'date' => ['2024-01-01', type_date()], + 'boolean' => ['true', type_boolean()], + 'timezone' => ['UTC', type_time_zone()], +]; +$candidates = array_map(static fn($type): string => $type->toString(), InferredTypes::default()->toArray()); +$widener = new TypeWidener(); +$pairs = 0; +$mismatches = 0; + +foreach ($leaves as $left => [$leftCell, $leftType]) { + foreach ($leaves as $right => [$rightCell, $rightType]) { + $csv = fopen('php://memory', 'rb+'); + fputcsv($csv, ['c'], ',', '"', '\\'); + fputcsv($csv, [$leftCell ?? ''], ',', '"', '\\'); + fputcsv($csv, [$rightCell ?? ''], ',', '"', '\\'); + rewind($csv); + + $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); + $reader->feed((string) stream_get_contents($csv)); + $reader->finish(); + $fold = new RustColumnFoldNative([], $candidates); + $reader->fold($fold, -1); + + $expected = $widener->widen($leftType, $rightType)->toString(); + $actual = $fold->types()['c']; + $pairs++; + + if ($expected !== $actual) { + $mismatches++; + echo "{$left} + {$right}: TypeWidener={$expected} native={$actual}\n"; + } + } +} + +echo "{$pairs} pairs, {$mismatches} mismatches\n"; +?> +--EXPECT-- +100 pairs, 0 mismatches diff --git a/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt new file mode 100644 index 0000000000..97fb9c6374 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt @@ -0,0 +1,43 @@ +--TEST-- +repeated native CSV folding - including every PHP callback and a rejected time zone offset - does not leak memory +--SKIPIF-- + +--FILE-- + $type->toString(), InferredTypes::default()->toArray()); +$raw = "id,json,at,zone,offset,dup,dup\n1,\"{\"\"a\"\":1}\",2024-01-01 10:00:00,Europe/Warsaw,+99:60,x,\n2,[1],2024-02-30,UTC,+02:00,,y\n"; + +$cycle = static function () use ($candidates, $raw): void { + $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); + $reader->feed($raw); + $reader->finish(); + $fold = new RustColumnFoldNative(['id', 'json'], $candidates); + $reader->fold($fold, -1); + $fold->types(); + $fold->rows(); + $fold->narrowOne('+99:60'); + $fold->narrowOne('{"a":[1,2]}'); + $fold->narrowOne('2024-01-01'); +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 10000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt b/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt new file mode 100644 index 0000000000..6ccff95294 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/050_csv_tokenizer_fuzz.phpt @@ -0,0 +1,54 @@ +--TEST-- +native CSV rows match the PHP path over a seeded fuzz corpus, every dialect and chunk size +--SKIPIF-- + +--FILE-- + [',', "a, \"b\nc\",d\ne\n"], + 'whitespace separator before an enclosure' => ["\t", "h1\th2\th3\na\t\t\"b\"\n"], + 'trailing carriage return in an unenclosed field' => [',', "h1,h2\na\r,b\n"], +]; + +foreach ($named as $case => [$separator, $raw]) { + assert_csv_identical( + $case, + csv_php_rows($raw, $separator, '"', '\\', false), + csv_native_rows($raw, $separator, '"', '\\', false), + ); +} + +mt_srand(50); +$alphabet = ['a', 'b', ' ', "\t", ',', ';', '"', "'", '\\', "\n", "\r"]; +$dialects = [[',', '"', '\\'], [';', '"', '\\'], [',', "'", '\\'], [',', '"', ''], ["\t", '"', '\\']]; +$mismatches = 0; + +for ($i = 0; $i < 2000; $i++) { + $raw = ''; + + for ($j = mt_rand(0, 40); $j > 0; $j--) { + $raw .= $alphabet[mt_rand(0, count($alphabet) - 1)]; + } + + [$separator, $enclosure, $escape] = $dialects[mt_rand(0, count($dialects) - 1)]; + $withHeader = mt_rand(0, 1) === 1; + $emptyToNull = mt_rand(0, 1) === 1; + $expected = csv_php_rows($raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull); + + foreach ([1, 7, 4096] as $chunk) { + if (csv_native_rows($raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull, true, $chunk) !== $expected) { + $mismatches++; + echo 'MISMATCH ', json_encode([$raw, $separator, $enclosure, $escape, $withHeader, $emptyToNull, $chunk]), "\n"; + } + } +} + +echo "fuzz mismatches: {$mismatches}\n"; +?> +--EXPECT-- +blanks before a multi-line enclosure: identical +whitespace separator before an enclosure: identical +trailing carriage return in an unenclosed field: identical +fuzz mismatches: 0 diff --git a/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt new file mode 100644 index 0000000000..9f97f6eb6e --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt @@ -0,0 +1,30 @@ +--TEST-- +native CSV classes reject an invalid dialect, batch size, fold limit and HTML/XML candidates +--SKIPIF-- + +--FILE-- + new RustCSVReaderNative(',,', '"', '\\', true, true, true)); +expect_exception(static fn() => new RustCSVReaderNative(',', '', '\\', true, true, true)); +expect_exception(static fn() => new RustCSVReaderNative(',', '"', '\\\\', true, true, true)); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(0)); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(-1)); +$fold = new RustColumnFoldNative([], ['integer']); +expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->fold($fold, -2)); +expect_exception(static fn() => new RustColumnFoldNative([], ['integer', 'html'])); +expect_exception(static fn() => new RustColumnFoldNative([], ['xml'])); +?> +--EXPECT-- +Flow\Floe\Exception\ExtensionException: flow_php CSV separator must be exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV enclosure must be exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV escape must be empty or exactly one byte +Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 +Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 +Flow\Floe\Exception\ExtensionException: flow_php CSV fold limit must be -1 or at least 0 +Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively +Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively diff --git a/src/extension/flow-php-ext/tests/phpt/bootstrap.php b/src/extension/flow-php-ext/tests/phpt/bootstrap.php index 0913d9121c..e3f35eb67c 100644 --- a/src/extension/flow-php-ext/tests/phpt/bootstrap.php +++ b/src/extension/flow-php-ext/tests/phpt/bootstrap.php @@ -175,3 +175,297 @@ function expect_exception(callable $fn): void echo get_class($e), ': ', $e->getMessage(), "\n"; } } + +/** + * The PHP CSV path - CSVLineReader + CSVEncoder::decode() - as `[headers, list of RawRowValues::$values]`. + * + * @return array{list, list>} + */ +function csv_php_rows( + string $raw, + string $separator, + string $enclosure, + string $escape, + bool $withHeader = true, + bool $emptyToNull = true, + bool $removeBOM = true, +): array { + $encoder = new Flow\ETL\Adapter\CSV\CSVEncoder( + withHeader: $withHeader, + separator: $separator, + enclosure: $enclosure, + escape: $escape, + emptyToNull: $emptyToNull, + ); + $lines = new Flow\ETL\Adapter\CSV\CSVLineReader($enclosure, $separator, $escape, removeBOM: $removeBOM); + $rows = []; + + foreach ($lines->readLines( + new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.csv'), $raw), + ) as $line) { + foreach ($encoder->decode([$line]) as $values) { + $rows[] = $values->values; + } + } + + return [$encoder->headers() ?? [], $rows]; +} + +/** + * RustCSVReaderNative fed `$raw` in `$chunk`-byte pieces, as `[headers, list of RawRowValues::$values]`. + * + * @param positive-int $chunk + * + * @return array{list, list>} + */ +function csv_native_rows( + string $raw, + string $separator, + string $enclosure, + string $escape, + bool $withHeader = true, + bool $emptyToNull = true, + bool $removeBOM = true, + int $chunk = 4096, +): array { + $reader = new Flow\ETL\Adapter\CSV\RustCSVReaderNative( + $separator, + $enclosure, + $escape, + $withHeader, + $emptyToNull, + $removeBOM, + ); + $rows = []; + $drain = static function () use ($reader, &$rows): void { + while (($batch = $reader->next(3)) !== []) { + foreach ($batch as $values) { + $rows[] = $values->values; + } + } + }; + + foreach ($raw === '' ? [] : str_split($raw, $chunk) as $piece) { + $reader->feed($piece); + $drain(); + } + + $reader->finish(); + $drain(); + + return [$reader->headers(), $rows]; +} + +/** + * Prints `