Fix #14688 (test-my-pr.py: run against the debian ftp server directly) (#8491)

Author: danmar

tools/test-my-pr.py

@@ -8,9 +8,13 @@
 import donate_cpu_lib as lib
 import argparse
 import glob
+import gzip
+import natsort
 import os
 import sys
 import random
+import re
+import requests
 import subprocess
 
 
@@ -20,6 +24,100 @@ def format_float(a, b=1):
     return 'N/A'
 
 
+def ftp_get(url):
+    try:
+        response = requests.get(url, timeout=300)
+        response.raise_for_status()
+        return response.content
+    except requests.RequestException as err:
+        print('Failed to fetch {}: {}'.format(url, err))
+    return None
+
+
+def latestvername(names):
+    s = natsort.natsorted(names, key=lambda x: x[x.index('_')+1:x.index('.orig.tar')])
+    return s[-1]
+
+
+def getpackages():
+    debian = 'https://ftp.debian.org/debian/'
+
+    data = ftp_get(debian + 'ls-lR.gz')
+    if data is None:
+        print('Failed to fetch ls-lR.gz')
+        sys.exit(1)
+
+    lines = gzip.decompress(data).decode('utf-8', errors='replace').splitlines()
+
+    # Example content in ls-lR:
+    #./pool/main/0/0xffff:
+    #total 1452
+    #-rw-r--r-- 2 dak debadmin  6524 Dec 25  2016 0xffff_0.7-2.debian.tar.xz
+    #-rw-r--r-- 2 dak debadmin  1791 Dec 25  2016 0xffff_0.7-2.dsc
+    #-rw-r--r-- 2 dak debadmin 57168 Dec 25  2016 0xffff_0.7-2_amd64.deb
+    #-rw-r--r-- 2 dak debadmin 48578 Dec 26  2016 0xffff_0.7-2_arm64.deb
+    #-rw-r--r-- 2 dak debadmin 56730 Dec 26  2016 0xffff_0.7-2_armel.deb
+    #-rw-r--r-- 2 dak debadmin 57296 Dec 26  2016 0xffff_0.7-2_armhf.deb
+    #-rw-r--r-- 2 dak debadmin 60254 Dec 26  2016 0xffff_0.7-2_i386.deb
+    #-rw-r--r-- 2 dak debadmin 53130 Dec 26  2016 0xffff_0.7-2_mips.deb
+    #-rw-r--r-- 2 dak debadmin 52542 Dec 26  2016 0xffff_0.7-2_mips64el.deb
+    #-rw-r--r-- 2 dak debadmin 53712 Dec 26  2016 0xffff_0.7-2_mipsel.deb
+    #-rw-r--r-- 2 dak debadmin 51908 Dec 26  2016 0xffff_0.7-2_ppc64el.deb
+    #-rw-r--r-- 2 dak debadmin 53548 Dec 26  2016 0xffff_0.7-2_s390x.deb
+    #-rw-r--r-- 2 dak debadmin 65248 Dec 25  2016 0xffff_0.7.orig.tar.gz
+    #-rw-r--r-- 2 dak debadmin  6884 Jul 19 19:08 0xffff_0.8-1.debian.tar.xz
+    #-rw-r--r-- 2 dak debadmin  1807 Jul 19 19:08 0xffff_0.8-1.dsc
+    #-rw-r--r-- 2 dak debadmin 58908 Jul 19 19:08 0xffff_0.8-1_amd64.deb
+    #-rw-r--r-- 2 dak debadmin 51340 Jul 19 19:58 0xffff_0.8-1_arm64.deb
+    #-rw-r--r-- 2 dak debadmin 57612 Jul 19 20:13 0xffff_0.8-1_armel.deb
+    #-rw-r--r-- 2 dak debadmin 58584 Jul 19 19:58 0xffff_0.8-1_armhf.deb
+    #-rw-r--r-- 2 dak debadmin 57544 Jul 19 20:23 0xffff_0.8-1_hurd-i386.deb
+    #-rw-r--r-- 2 dak debadmin 62048 Jul 19 23:54 0xffff_0.8-1_i386.deb
+    #-rw-r--r-- 2 dak debadmin 55080 Jul 23 19:07 0xffff_0.8-1_kfreebsd-amd64.deb
+    #-rw-r--r-- 2 dak debadmin 58392 Jul 23 19:07 0xffff_0.8-1_kfreebsd-i386.deb
+    #-rw-r--r-- 2 dak debadmin 54144 Jul 19 22:28 0xffff_0.8-1_mips.deb
+    #-rw-r--r-- 2 dak debadmin 53648 Jul 20 00:56 0xffff_0.8-1_mips64el.deb
+    #-rw-r--r-- 2 dak debadmin 54740 Jul 19 22:58 0xffff_0.8-1_mipsel.deb
+    #-rw-r--r-- 2 dak debadmin 57424 Jul 19 19:58 0xffff_0.8-1_ppc64el.deb
+    #-rw-r--r-- 2 dak debadmin 53764 Jul 19 22:28 0xffff_0.8-1_s390x.deb
+    #-rw-r--r-- 2 dak debadmin 64504 Jul 19 19:08 0xffff_0.8.orig.tar.gz
+    #
+
+    path = None
+    previous_path = ''
+    archives = []
+    filename = None
+    filenames = []
+    for line in lines:
+        line = line.strip()
+        if len(line) < 4:
+            if filename:
+                res1 = re.match(r'(.*)-[0-9.]+$', path)
+                if res1 is None:
+                    res1 = re.match(r'(.*)[-.][0-9.]+$', path)
+                res2 = re.match(r'(.*)-[0-9.]+$', previous_path)
+                if res2 is None:
+                    res2 = re.match(r'(.*)[-.][0-9.]+$', previous_path)
+                if res1 is None or res2 is None or res1.group(1) != res2.group(1):
+                    archives.append(path + '/' + latestvername(filenames))
+                else:
+                    archives[-1] = path + '/' + latestvername(filenames)
+            if path:
+                previous_path = path
+            path = None
+            filename = None
+            filenames = []
+        elif line.startswith('./pool/main/'):
+            path = debian + line[2:-1]
+        elif path and line.endswith(('.orig.tar.gz', '.orig.tar.bz2', '.orig.tar.xz')):
+            filename = line[1 + line.rfind(' '):]
+            filenames.append(filename)
+
+    return archives
+
+
+
+
 if __name__ == "__main__":
     __my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
     __work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder'))
@@ -40,6 +138,16 @@ def format_float(a, b=1):
 
     print(args)
 
+    if args.packages_path:
+        # You can download packages using daca2-download.py
+        args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
+        random.shuffle(args.packages)
+    elif args.packages is None:
+        args.packages = getpackages()
+        random.shuffle(args.packages)
+
+    print('\n'.join(args.packages[:20]))
+
     if not lib.check_requirements():
         print("Error: Check requirements")
         sys.exit(1)
@@ -100,35 +208,14 @@ def format_float(a, b=1):
         print('Failed to compile your version of Cppcheck')
         sys.exit(1)
 
-    if args.packages_path:
-        # You can download packages using daca2-download.py
-        args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
-        args.p = len(args.packages)
-        packages_idxs = list(range(args.p))
-        random.shuffle(packages_idxs)
-    elif args.packages:
-        args.p = len(args.packages)
-        packages_idxs = []
-    else:
-        packages_count = lib.get_packages_count()
-        if not packages_count:
-            print("network or server might be temporarily down..")
-            sys.exit(1)
-
-        packages_idxs = list(range(packages_count))
-        random.shuffle(packages_idxs)
-
     packages_processed = 0
     crashes = []
     timeouts = []
 
-    while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages:
-        if args.packages:
-            package = args.packages.pop()
-        else:
-            package = lib.get_package(packages_idxs.pop())
+    while packages_processed < args.p and args.packages:
+        package = args.packages.pop()
 
-        if package.startswith('ftp://') or package.startswith('http://'):
+        if package.startswith('ftp://') or package.startswith('https://'):
             tgz = lib.download_package(work_path, package, None)
             if tgz is None:
                 print("No package downloaded")