Merge remote-tracking branch 'origin/master' into fix/sync_loop_for_too_long_path

Author: harminius

.github/workflows/security_check.yml

@@ -1,42 +1,16 @@
-name: Python-api QA (Security & Style)
+name: QGIS Plugin Repository Checks
 
-# Trigger the workflow on every push
 on: [push]
 
 jobs:
-  quality-assurance:
+  qgis_repo_checks:
+    name: Mirror plugins.qgis.org upload scan
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      # This package ships inside the QGIS plugin zip, so the scan that gates it is
+      # the plugin's. The action lives there to keep one copy of the rule snapshot.
+      - uses: MerginMaps/qgis-plugin/.github/actions/qgis-repo-checks@master
         with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          # Upgrade pip and install security/linting tools
-          python -m pip install --upgrade pip
-          pip install bandit detect-secrets
-
-      # - name: Install dependencies
-      #   run: |
-      #     # Upgrade pip and install security/linting tools
-      #     python -m pip install --upgrade pip
-      #     pip install bandit detect-secrets flake8 flake8-json
-
-      - name: Run Bandit (Security Scan)
-        # Scan the mergin folder for vulnerabilities, excluding the test directory
-        run: bandit -r ./mergin/ -ll --exclude ./mergin/test
-
-      - name: Run Detect Secrets
-        # Scan the plugin directory for hardcoded secrets/credentials
-        run: detect-secrets scan ./mergin/ --all-files
-
-      # - name: Run Flake8 (Style Check)
-      #   # Style enforcement using MerginMaps standards
-      #   # Ignoring E501 (line length) and W503 (operator line breaks)
-      #   run: |
-      #     flake8 ./mergin/ --max-line-length=120 --ignore=E501,W503 --exclude=test
+          root: mergin

.gitignore

@@ -14,3 +14,4 @@ deps
 venv
 debug.py
 .vscode/
+.python-version

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+# Black version is kept in sync with .github/workflows/code_style.yml
+repos:
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 26.1.0
+    hooks:
+      - id: black
+        args: ["--line-length=120"]

CITATION.cff

@@ -0,0 +1,9 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "Mergin Maps Python Client"
+type: software
+authors:
+  - name: "Lutra Consulting Ltd."
+url: "https://pypi.org/project/mergin-client/"
+repository-code: "https://github.com/MerginMaps/python-api-client"
+license: MIT

README.md

@@ -9,8 +9,7 @@
 
 This repository contains a Python client module for access to [Mergin Maps](https://merginmaps.com/) service and a command-line tool for easy access to data stored in Mergin Maps.
 
-<div><img align="left" width="45" height="45" src="https://raw.githubusercontent.com/MerginMaps/docs/main/src/public/slack.svg"><a href="https://merginmaps.com/community/join">Join our community chat</a><br/>and ask questions!</div><br />
-
+<a href="https://community.merginmaps.com">Join our community</a> and ask questions!
 
 To install the module:
 ```bash
@@ -192,3 +191,13 @@ For running test do:
     pip install pytest pytest-cov coveralls
     pytest --cov-report html --cov=mergin mergin/test/
 ```
+
+### Code style
+
+Code is formatted with [black](https://github.com/psf/black) (line length 120), enforced in CI.
+To run the same check automatically before each commit, install the [pre-commit](https://pre-commit.com/) hooks once:
+
+```bash
+    pip install pre-commit
+    pre-commit install
+```

mergin/cli.py

@@ -29,6 +29,7 @@
     download_project_cancel,
     download_file_async,
     download_file_finalize,
+    download_project_file_async,
     download_project_finalize,
     download_project_is_running,
 )
@@ -248,16 +249,20 @@ def list_projects(ctx, name, namespace, order_params):
 @click.argument("project")
 @click.argument("directory", type=click.Path(), required=False)
 @click.option("--version", default=None, help="Version of project to download")
+@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'")
+@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'")
 @click.pass_context
-def download(ctx, project, directory, version):
+def download(ctx, project, directory, version, include, exclude):
     """Download last version of mergin project."""
     mc = ctx.obj["client"]
     if mc is None:
         return
+    if include and exclude:
+        raise click.UsageError("--include and --exclude cannot be used together")
     directory = directory or os.path.basename(project)
     click.echo("Downloading into {}".format(directory))
     try:
-        job = download_project_async(mc, project, directory, version)
+        job = download_project_async(mc, project, directory, version, include=include, exclude=exclude)
         with click.progressbar(length=job.total_size) as bar:
             last_transferred_size = 0
             while download_project_is_running(job):
@@ -335,17 +340,33 @@ def share(ctx, project):
 @click.argument("filepath")
 @click.argument("output")
 @click.option("--version", help="Project version tag, for example 'v3'")
+@click.option(
+    "--project",
+    help="Full project name ('<workspace>/<project>') to download the file directly from the server. "
+    "If not given, the current directory is used and must be an existing checked out project.",
+)
 @click.pass_context
-def download_file(ctx, filepath, output, version):
+def download_file(ctx, filepath, output, version, project):
     """
-    Download project file at specified version. `project` needs to be a combination of namespace/project.
-    If no version is given, the latest will be fetched.
+    Download project file at specified version. If no version is given, the latest will be fetched.
     """
     mc = ctx.obj["client"]
     if mc is None:
         return
     try:
-        job = download_file_async(mc, os.getcwd(), filepath, output, version)
+        if project is not None:
+            job = download_project_file_async(mc, project, filepath, output, version)
+        else:
+            try:
+                MerginProject(os.getcwd()).project_full_name()
+            except InvalidProject:
+                click.secho(
+                    "Current directory is not a Mergin Maps project. Run this command from within a "
+                    "checked out project directory, or pass --project <workspace>/<project>.",
+                    fg="red",
+                )
+                return
+            job = download_file_async(mc, os.getcwd(), filepath, output, version)
         with click.progressbar(length=job.total_size) as bar:
             last_transferred_size = 0
             while download_project_is_running(job):

mergin/client.py

@@ -45,6 +45,7 @@
     download_file_async,
     download_files_async,
     download_files_finalize,
+    download_project_file_async,
     download_diffs_async,
     download_project_finalize,
     download_project_wait,
@@ -62,6 +63,7 @@
 from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable
 from .utils import (
     DateTimeEncoder,
+    filter_files,
     get_versions_with_file_changes,
     int_version,
     is_version_acceptable,
@@ -903,7 +905,7 @@ def project_versions(self, project_path, since=1, to=None):
         filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions))
         return filtered_versions
 
-    def download_project(self, project_path, directory, version=None):
+    def download_project(self, project_path, directory, version=None, include=None, exclude=None):
         """
         Download project into given directory. If version is not specified, latest version is downloaded
 
@@ -915,8 +917,16 @@ def download_project(self, project_path, directory, version=None):
 
         :param version: Project version to download, e.g. v42
         :type version: String
+
+        :param include: Optional list of glob patterns (matched against each file's project path, e.g.
+            "media/*" or "*.gpkg") - only matching files are downloaded.
+        :type include: List[String]
+
+        :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive
+            with include.
+        :type exclude: List[String]
         """
-        job = download_project_async(self, project_path, directory, version)
+        job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude)
         download_project_wait(job)
         download_project_finalize(job)
 
@@ -1158,7 +1168,13 @@ def project_status(self, directory):
         mp = MerginProject(directory)
         server_info = self.project_info(mp.project_full_name(), since=mp.version())
 
+        mp.update_project_role(server_info["role"])
+
         pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version"))
+        # on a sparse checkout, don't report excluded files as pending server changes -
+        # they were never meant to be pulled in the first place
+        file_filter = mp.file_filter()
+        pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()}
 
         push_changes = mp.get_push_changes()
         push_changes_summary = mp.get_list_of_push_changes(push_changes)
@@ -1213,6 +1229,24 @@ def download_file(self, project_dir, file_path, output_filename, version=None):
         pull_project_wait(job)
         download_file_finalize(job)
 
+    def download_project_file(self, project_path, file_path, output_filename, version=None):
+        """
+        Download a single project file at specified version directly from the server, without
+        needing an existing local project checkout.
+
+        :param project_path: full project name ("<workspace>/<project>")
+        :type project_path: String
+        :param file_path: relative path of file to download in the project directory
+        :type file_path: String
+        :param output_filename: full destination path for saving the downloaded file
+        :type output_filename: String
+        :param version: optional version tag for downloaded file
+        :type version: String
+        """
+        job = download_project_file_async(self, project_path, file_path, output_filename, version=version)
+        pull_project_wait(job)
+        download_file_finalize(job)
+
     def get_file_diff(self, project_dir, file_path, output_diff, version_from, version_to):
         """Create concatenated diff for project file diffs between versions version_from and version_to.
 
@@ -1241,7 +1275,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi
             diffs.append(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v))
 
         # concatenate diffs, if needed
-        output_dir = os.path.dirname(output_diff)
+        output_dir = os.path.abspath(os.path.dirname(output_diff))
         if len(diffs) >= 1:
             fs.makedirs(output_dir, exist_ok=True)
             if len(diffs) > 1: