Antalya 25.8 - replace azure with azurite

ci/jobs/functional_tests.py

@@ -190,27 +190,6 @@ def main():
 
     runner_options += f" --jobs {nproc}"
 
-    if not info.is_local_run:
-        # NOTE(strtgbb): We pass azure credentials through the docker command, not SSM.
-        # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
-        # os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
-        #     f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
-        #     verbose=True,
-        # )
-
-        # NOTE(strtgbb): Azure credentials don't exist in community workflow
-        if info.is_community_pr:
-            print(
-                "NOTE: No azure credentials provided for community PR - disable azure storage"
-            )
-            config_installs_args += " --no-azure"
-
-            # NOTE(strtgbb): With the above, some tests are still trying to use azure, try this:
-            os.environ["USE_AZURE_STORAGE_FOR_MERGE_TREE"] = "0"
-    else:
-        print("Disable azure for a local run")
-        config_installs_args += " --no-azure"
-
     if (is_azure_storage or is_s3_storage) and is_encrypted_storage:
         config_installs_args += " --encrypted-storage"
         runner_options += f" --encrypted-storage"

ci/jobs/scripts/clickhouse_proc.py

@@ -153,7 +153,19 @@ def start_azurite(self):
                 command, stdout=log_file, stderr=subprocess.STDOUT, shell=True
             )
         print(f"Started azurite asynchronously with PID {self.azurite_proc.pid}")
-        return True
+
+        print("Waiting for azurite to start...")
+        for _ in range(10):
+            res = Shell.check(
+                "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'",
+                verbose=False,
+            )
+            if res:
+                print("Azurite started successfully")
+                return True
+            time.sleep(3)
+        print("Failed to start azurite")
+        return False
 
     @staticmethod
     def log_cluster_config():
@@ -1048,6 +1060,8 @@ def set_random_timezone():
             param = sys.argv[2]
             assert param in ["stateless"]
             res = ch.start_minio(param)
+        elif command == "start_azurite":
+            res = ch.start_azurite()
         else:
             raise ValueError(f"Unknown command: {command}")
     except Exception as e:

tests/config/config.d/azure_storage_conf.xml

@@ -6,10 +6,8 @@
                 <object_storage_type>azure</object_storage_type>
                 <skip_access_check>false</skip_access_check>
                 <max_single_part_upload_size>33554432</max_single_part_upload_size>
-                <storage_account_url from_env="AZURE_STORAGE_ACCOUNT_URL"/>
-                <container_name from_env="AZURE_CONTAINER_NAME"/>
-                <account_name from_env="AZURE_ACCOUNT_NAME"/>
-                <account_key from_env="AZURE_STORAGE_KEY"/>
+                <container_name>clickhouse-tests</container_name>
+                <connection_string>DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;</connection_string>
             </azure>
             <cached_azure>
                 <type>cache</type>

tests/docker_scripts/stress_runner.sh

@@ -57,6 +57,7 @@ configure
 cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py logs_export_config || echo "ERROR: Failed to create log export config"
 
 cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || { echo "Failed to start minio"; exit 1; }
+cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; }
 
 start_server || { echo "Failed to start server"; exit 1; }