Add broken tags to FW's that haven't started/stopped correctly in last 3 full runs + script for detecting failing frameworks

frameworks/Java/spring-webflux/benchmark_config.json

@@ -41,7 +41,8 @@
       "database_os": "Linux",
       "display_name": "spring-webflux-mongo",
       "notes": "",
-      "versus": "spring"
+      "versus": "spring",
+      "tags": ["broken"]
     },
     "pgclient": {
       "db_url": "/db",
@@ -62,7 +63,8 @@
       "database_os": "Linux",
       "display_name": "spring-webflux-pgclient",
       "notes": "",
-      "versus": "spring"
+      "versus": "spring",
+      "tags": ["broken"]
     },
     "rxjdbc": {
       "db_url": "/db",
@@ -83,7 +85,8 @@
       "database_os": "Linux",
       "display_name": "spring-webflux-rxjdbc",
       "notes": "",
-      "versus": "spring"
+      "versus": "spring",
+      "tags": ["broken"]
     },
     "jdbc": {
       "db_url": "/db",
@@ -104,7 +107,8 @@
       "database_os": "Linux",
       "display_name": "spring-webflux-jdbc",
       "notes": "",
-      "versus": "spring"
+      "versus": "spring",
+      "tags": ["broken"]
     }
   }]
 }

frameworks/JavaScript/sailsjs/benchmark_config.json

@@ -22,7 +22,8 @@
       "database_os": "Linux",
       "display_name": "Sails.js",
       "notes": "",
-      "versus": "nodejs"
+      "versus": "nodejs",
+      "tags": ["broken"]
     },
     "postgres": {
       "db_url": "/postgres/db",

frameworks/Kotlin/vertx-web-kotlin-coroutines/benchmark_config.json

@@ -18,7 +18,8 @@
       "database_os": "Linux",
       "display_name": "vertx-web-kotlin-coroutines",
       "notes": "",
-      "versus": "vertx-web"
+      "versus": "vertx-web",
+      "tags": ["broken"]
     },
     "postgres": {
       "db_url": "/db",

frameworks/PHP/php/benchmark_config.json

@@ -112,7 +112,8 @@
       "database_os": "Linux",
       "display_name": "PHP Unit Nginx",
       "notes": "",
-      "versus": "php"
+      "versus": "php",
+      "tags": ["broken"]
     },
     "pools": {
       "json_url": "/json.php",

frameworks/Python/cherrypy/app.py

@@ -154,6 +154,8 @@ def fortunes(self):
     cherrypy.server.socket_host = '0.0.0.0'
     cherrypy.server.socket_port = 8080
     cherrypy.server.thread_pool = workers
+    cherrypy.server.socket_queue_size = 25
+
     cherrypy.quickstart(CherryPyBenchmark(), '', {
         '/': {
             'tools.db.on': True,

frameworks/Python/cherrypy/benchmark_config.json

@@ -21,7 +21,8 @@
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "CherryPy [py2]",
-      "notes": "CPython 2.7"
+      "notes": "CPython 2.7",
+      "tags": ["broken"]
     },
     "py3": {
       "json_url": "/json",

frameworks/Python/fastapi/benchmark_config.json

@@ -169,7 +169,8 @@
         "database_os": "Linux",
         "display_name": "FastAPI-nginx-unit",
         "notes": "",
-        "versus": "None"
+        "versus": "None",
+        "tags": ["broken"]
       },
       "nginx-unit-orjson": {
         "json_url": "/json",
@@ -190,7 +191,8 @@
         "database_os": "Linux",
         "display_name": "FastAPI-nginx-unit-orjson",
         "notes": "",
-        "versus": "None"
+        "versus": "None",
+        "tags": ["broken"]
       },
       "uvicorn": {
         "json_url": "/json",

frameworks/Python/sanic/benchmark_config.json

@@ -23,7 +23,8 @@
         "database_os": "Linux",
         "display_name": "Sanic",
         "notes": "",
-        "versus": "None"
+        "versus": "None",
+        "tags": ["broken"]
       }
     }
   ]

frameworks/Rust/rocket/benchmark_config.json

@@ -22,7 +22,8 @@
         "database_os": "Linux",
         "display_name": "Rocket",
         "notes": "",
-        "versus": "None"
+        "versus": "None",
+        "tags": ["broken"]
       },
       "diesel": {
         "json_url": "/json",
@@ -45,7 +46,8 @@
         "database_os": "Linux",
         "display_name": "Rocket (Diesel)",
         "notes": "",
-        "versus": "None"
+        "versus": "None",
+        "tags": ["broken"]
       }
     }]
 }

frameworks/Scala/snunit/benchmark_config.json

@@ -19,7 +19,8 @@
         "database_os": "Linux",
         "display_name": "SNUnit",
         "notes": "",
-        "versus": ""
+        "versus": "",
+        "tags": ["broken"]
       }
     }
   ]

scripts/tfb-fail-detector.py

@@ -0,0 +1,78 @@
+"""
+TFB Fail Detector Web Scraper
+Scrapes data from the last 3 complete runs and outputs the frameworks that have failed all of them with the same errors.
+
+Developed using selenium==4.10.0 and the selenium/standalone-chrome:4.10.0 docker image.
+
+USAGE: python tfb-fail-detector.py
+"""
+
+from collections import defaultdict
+import re
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+
+NUM_RUNS = 3
+
+def get_driver():
+  """Gets the selenium webdriver for interacting with the website"""
+  chrome_options = Options()
+  chrome_options.add_argument("--headless")
+  chrome_options.add_argument("--no-sandbox")
+  chrome_options.add_argument("--disable-dev-shm-usage")
+  chrome_prefs = {}
+  chrome_options.experimental_options["prefs"] = chrome_prefs
+  chrome_prefs["profile.default_content_settings"] = {"images": 2}
+  driver = webdriver.Chrome(options=chrome_options)
+  return driver
+
+def get_last_n_complete_runs(driver, n=3):
+  """Scrape the last n complete runs"""
+  rows = driver.find_elements(By.CSS_SELECTOR, "table.resultsTable > tbody > tr")
+  complete_runs = []
+  for row in rows:
+    row_uuid = row.get_dom_attribute("data-uuid")
+    run_stats = row.find_element(By.CSS_SELECTOR, "td:nth-of-type(2)")
+    frameworks_tested_stats = re.search(r"([0-9]+)/([0-9]+) frameworks tested", run_stats.text)
+    if not frameworks_tested_stats:
+      # print("Unable to get info from run %s" % row_uuid)
+      continue
+    tested, total = frameworks_tested_stats.groups()
+    if tested != total:
+      # print("Found incomplete run %s. Tested: %s/%s" % (row_uuid, tested, total))
+      continue
+    # print("Found complete run %s. Tested: %s/%s" % (row_uuid, tested, total))
+    complete_runs.append(row_uuid)
+    if len(complete_runs) >= n:
+      return complete_runs
+
+def find_failing_frameworks(driver, run_uuids):
+  """Find frameworks that have failed all the given runs with the same error message"""
+  failing_frameworks = defaultdict(lambda: 0)
+  def process_failed_framework(framework_element):
+    framework = re.search(r"\[\S+] [a-zA-Z][a-zA-Z:\_ ]*$", framework_element.text)
+    framework_failure = framework.group(0)
+    failing_frameworks[framework_failure] += 1
+
+  for run_uuid in run_uuids:
+    driver.get("https://tfb-status.techempower.com/results/%s" % run_uuid)
+    assert "TFB Status" in driver.title
+    failure_list = driver.find_element(By.CLASS_NAME, "failures")
+    failures = failure_list.find_elements(By.TAG_NAME, "li")
+    for failure in failures:
+      process_failed_framework(failure)
+  return failing_frameworks
+
+if __name__ == '__main__':
+  driver = get_driver()
+  driver.get("https://tfb-status.techempower.com/")
+  assert "TFB Status" in driver.title
+  complete_runs = get_last_n_complete_runs(driver, NUM_RUNS)
+  failed_frameworks = find_failing_frameworks(driver, complete_runs)
+
+  for failure_info, fail_count in failed_frameworks.items():
+    if fail_count != NUM_RUNS:
+      continue
+    print(failure_info)