chore: enforce more ruff rules (#31447)

Co-authored-by: Elizabeth Thompson <[email protected]>

RELEASING/changelog.py

@@ -272,14 +272,14 @@ def __repr__(self) -> str:
 
     @staticmethod
     def _git_get_current_head() -> str:
-        output = os.popen("git status | head -1").read()
+        output = os.popen("git status | head -1").read()  # noqa: S605, S607
         match = re.match("(?:HEAD detached at|On branch) (.*)", output)
         if not match:
             return ""
         return match.group(1)
 
     def _git_checkout(self, git_ref: str) -> None:
-        os.popen(f"git checkout {git_ref}").read()
+        os.popen(f"git checkout {git_ref}").read()  # noqa: S605
         current_head = self._git_get_current_head()
         if current_head != git_ref:
             print(f"Could not checkout {git_ref}")
@@ -290,7 +290,7 @@ def _git_logs(self) -> list[str]:
         current_git_ref = self._git_get_current_head()
         self._git_checkout(self._git_ref)
         output = (
-            os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
+            os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')  # noqa: S605, S607
             .read()
             .split("\n")
         )

RELEASING/generate_email.py

@@ -31,7 +31,7 @@
 RECEIVER_EMAIL = "[email protected]"
 PROJECT_NAME = "Superset"
 PROJECT_MODULE = "superset"
-PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application."
+PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application."  # noqa: E501
 
 
 def string_comma_to_list(message: str) -> list[str]:

RELEASING/verify_release.py

@@ -23,12 +23,12 @@
 
 import requests
 
-# Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512`
+# Part 1: Verify SHA512 hash - this is the same as running `shasum -a 512 {release}` and comparing it against `{release}.sha512`  # noqa: E501
 
 
 def get_sha512_hash(filename: str) -> str:
     """Run the shasum command on the file and return the SHA512 hash."""
-    result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE)
+    result = subprocess.run(["shasum", "-a", "512", filename], stdout=subprocess.PIPE)  # noqa: S603, S607
     sha512_hash = result.stdout.decode().split()[0]
     return sha512_hash
 
@@ -43,7 +43,7 @@ def read_sha512_file(filename: str) -> str:
 
 
 def verify_sha512(filename: str) -> str:
-    """Verify if the SHA512 hash of the file matches with the hash in the .sha512 file."""
+    """Verify if the SHA512 hash of the file matches with the hash in the .sha512 file."""  # noqa: E501
     sha512_hash = get_sha512_hash(filename)
     sha512_file_content = read_sha512_file(filename)
 
@@ -53,14 +53,15 @@ def verify_sha512(filename: str) -> str:
         return "SHA failed"
 
 
-# Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file
+# Part 2: Verify RSA key - this is the same as running `gpg --verify {release}.asc {release}` and comparing the RSA key and email address against the KEYS file  # noqa: E501
 
 
 def get_gpg_info(filename: str) -> tuple[Optional[str], Optional[str]]:
     """Run the GPG verify command and extract RSA key and email address."""
     asc_filename = filename + ".asc"
-    result = subprocess.run(
-        ["gpg", "--verify", asc_filename, filename], capture_output=True
+    result = subprocess.run(  # noqa: S603
+        ["gpg", "--verify", asc_filename, filename],  # noqa: S607
+        capture_output=True,  # noqa: S607
     )
     output = result.stderr.decode()
 
@@ -90,7 +91,7 @@ def get_gpg_info(filename: str) -> tuple[Optional[str], Optional[str]]:
 def verify_key(key: str, email: Optional[str]) -> str:
     """Fetch the KEYS file and verify if the RSA/EDDSA key and email match."""
     url = "https://downloads.apache.org/superset/KEYS"
-    response = requests.get(url)
+    response = requests.get(url)  # noqa: S113
     if response.status_code == 200:
         if key not in response.text:
             return "RSA/EDDSA key not found on KEYS page"

docker/pythonpath_dev/superset_config.py

@@ -99,7 +99,7 @@ class CeleryConfig:
 
 FEATURE_FLAGS = {"ALERT_REPORTS": True}
 ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
-WEBDRIVER_BASEURL = "http://superset:8088/"  # When using docker compose baseurl should be http://superset_app:8088/
+WEBDRIVER_BASEURL = "http://superset:8088/"  # When using docker compose baseurl should be http://superset_app:8088/  # noqa: E501
 # The base URL for the email report hyperlinks.
 WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
 SQLLAB_CTAS_NO_LIMIT = True

pyproject.toml

@@ -24,7 +24,7 @@ name = "apache-superset"
 description = "A modern, enterprise-ready business intelligence web application"
 readme = "README.md"
 dynamic = ["version", "scripts", "entry-points"]
-requires-python = "~=3.9"
+requires-python = ">=3.9"
 license = { file="LICENSE.txt" }
 authors = [
     { name = "Apache Software Foundation", email = "[email protected]" },
@@ -276,8 +276,8 @@ exclude = [
 line-length = 88
 indent-width = 4
 
-# Assume Python 3.8
-target-version = "py310"
+# Assume Python 3.9
+target-version = "py39"
 
 [tool.ruff.lint]
 # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
@@ -290,22 +290,24 @@ select = [
     "E9",
     "PT009",
     "TRY201",
-    # TODO add these rules in follow up PR
-    # "B",
-    # "C",
-    # "E",
-    # "F",
-    #"F",
-    # "I",
-    # "N",
-    # "PT",
-    # "Q",
-    # "S",
-    # "T",
-    #"W",
+    "B",
+    "C",
+    "E",
+    "F",
+    "F",
+    "I",
+    "N",
+    "PT",
+    "Q",
+    "S",
+    "T",
+    "W",
 ]
 ignore = [
     "S101",
+    "PT006",
+    "T201",
+    "N999",
 ]
 
 extend-select = ["I"]

scripts/benchmark_migration.py

@@ -70,7 +70,7 @@ def extract_modified_tables(module: ModuleType) -> set[str]:
     return tables
 
 
-def find_models(module: ModuleType) -> list[type[Model]]:
+def find_models(module: ModuleType) -> list[type[Model]]:  # noqa: C901
     """
     Find all models in a migration script.
     """
@@ -94,7 +94,7 @@ def find_models(module: ModuleType) -> list[type[Model]]:
     # downgrade
     sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"]
     engine = create_engine(sqlalchemy_uri)
-    Base = automap_base()
+    Base = automap_base()  # noqa: N806
     Base.prepare(engine, reflect=True)
     seen = set()
     while tables:
@@ -138,7 +138,7 @@ def find_models(module: ModuleType) -> list[type[Model]]:
 @click.option("--limit", default=1000, help="Maximum number of entities.")
 @click.option("--force", is_flag=True, help="Do not prompt for confirmation.")
 @click.option("--no-auto-cleanup", is_flag=True, help="Do not remove created models.")
-def main(
+def main(  # noqa: C901
     filepath: str, limit: int = 1000, force: bool = False, no_auto_cleanup: bool = False
 ) -> None:
     auto_cleanup = not no_auto_cleanup

scripts/cancel_github_workflows.py

@@ -49,7 +49,7 @@
 def request(
     method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kwargs: Any
 ) -> dict[str, Any]:
-    resp = requests.request(
+    resp = requests.request(  # noqa: S113
         method,
         f"https://api.github.com/{endpoint.lstrip('/')}",
         headers={"Authorization": f"Bearer {github_token}"},
@@ -152,7 +152,7 @@ def print_commit(commit: dict[str, Any], branch: str) -> None:
     help="Whether to also cancel running workflows.",
 )
 @click.argument("branch_or_pull", required=False)
-def cancel_github_workflows(
+def cancel_github_workflows(  # noqa: C901
     branch_or_pull: Optional[str],
     repo: str,
     event: list[str],

scripts/change_detector.py

@@ -51,12 +51,12 @@
 
 def fetch_files_github_api(url: str):  # type: ignore
     """Fetches data using GitHub API."""
-    req = Request(url)
+    req = Request(url)  # noqa: S310
     req.add_header("Authorization", f"Bearer {GITHUB_TOKEN}")
     req.add_header("Accept", "application/vnd.github.v3+json")
 
     print(f"Fetching from {url}")
-    with urlopen(req) as response:
+    with urlopen(req) as response:  # noqa: S310
         body = response.read()
         return json.loads(body)
 
@@ -130,7 +130,7 @@ def main(event_type: str, sha: str, repo: str) -> None:
         )
 
     # Output results
-    output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt"
+    output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt"  # noqa: S108
     with open(output_path, "a") as f:
         for check, changed in changes_detected.items():
             if changed:
@@ -139,8 +139,8 @@ def main(event_type: str, sha: str, repo: str) -> None:
 
 
 def get_git_sha() -> str:
-    return os.getenv("GITHUB_SHA") or subprocess.check_output(
-        ["git", "rev-parse", "HEAD"]
+    return os.getenv("GITHUB_SHA") or subprocess.check_output(  # noqa: S603
+        ["git", "rev-parse", "HEAD"]  # noqa: S607
     ).strip().decode("utf-8")
 
 

scripts/check-env.py

@@ -47,7 +47,7 @@ def __init__(
 
     def get_version(self) -> Optional[str]:
         try:
-            version = subprocess.check_output(self.command, shell=True).decode().strip()
+            version = subprocess.check_output(self.command, shell=True).decode().strip()  # noqa: S602
             if self.version_post_process:
                 version = self.version_post_process(version)
             return version.split()[-1]
@@ -76,7 +76,7 @@ def check_version(self) -> str:
     def format_result(self) -> str:
         ideal_range_str = f"{self.ideal_range[0]} - {self.ideal_range[1]}"
         supported_range_str = f"{self.supported_range[0]} - {self.supported_range[1]}"
-        return f"{self.status.split()[0]} {self.name:<25} {self.version or 'N/A':<25} {ideal_range_str:<25} {supported_range_str:<25}"
+        return f"{self.status.split()[0]} {self.name:<25} {self.version or 'N/A':<25} {ideal_range_str:<25} {supported_range_str:<25}"  # noqa: E501
 
 
 def check_memory(min_gb: int) -> str:
@@ -101,8 +101,9 @@ def get_cpu_info() -> str:
 def get_docker_platform() -> str:
     try:
         output = (
-            subprocess.check_output(
-                "docker info --format '{{.OperatingSystem}}'", shell=True
+            subprocess.check_output(  # noqa: S602
+                "docker info --format '{{.OperatingSystem}}'",  # noqa: S607
+                shell=True,  # noqa: S607
             )
             .decode()
             .strip()
@@ -117,7 +118,7 @@ def get_docker_platform() -> str:
 @click.command(
     help="""
 This script checks the local environment for various software versions and other requirements, providing feedback on whether they are ideal, supported, or unsupported.
-"""
+"""  # noqa: E501
 )
 @click.option(
     "--docker", is_flag=True, help="Check Docker and Docker Compose requirements"
@@ -128,7 +129,7 @@ def get_docker_platform() -> str:
     help="Check frontend requirements (npm, Node.js, memory)",
 )
 @click.option("--backend", is_flag=True, help="Check backend requirements (Python)")
-def main(docker: bool, frontend: bool, backend: bool) -> None:
+def main(docker: bool, frontend: bool, backend: bool) -> None:  # noqa: C901
     requirements = [
         Requirement(
             "python",

scripts/cypress_run.py

@@ -74,7 +74,7 @@ def run_cypress_for_test_file(
             print(f"DRY RUN: {cmd}")
             return 0
 
-        process = subprocess.Popen(
+        process = subprocess.Popen(  # noqa: S602
             cmd,
             shell=True,
             stdout=subprocess.PIPE,

scripts/erd/erd.py

@@ -171,7 +171,7 @@ def generate_erd(file_path: str) -> None:
     """
     data = introspect_models()
     templates_path = os.path.dirname(__file__)
-    env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))  # noqa: S701
 
     # Load the template
     template = env.get_template("erd.template.puml")

setup.py

@@ -30,7 +30,7 @@
 
 def get_git_sha() -> str:
     try:
-        output = subprocess.check_output(["git", "rev-parse", "HEAD"])
+        output = subprocess.check_output(["git", "rev-parse", "HEAD"])  # noqa: S603, S607
         return output.decode().strip()
     except Exception:  # pylint: disable=broad-except
         return ""
@@ -58,7 +58,7 @@ def get_git_sha() -> str:
     zip_safe=False,
     entry_points={
         "console_scripts": ["superset=superset.cli.main:superset"],
-        # the `postgres` and `postgres+psycopg2://` schemes were removed in SQLAlchemy 1.4
+        # the `postgres` and `postgres+psycopg2://` schemes were removed in SQLAlchemy 1.4  # noqa: E501
         # add an alias here to prevent breaking existing databases
         "sqlalchemy.dialects": [
             "postgres.psycopg2 = sqlalchemy.dialects.postgresql:dialect",

superset/advanced_data_type/api.py

@@ -39,7 +39,7 @@ class AdvancedDataTypeRestApi(BaseSupersetApi):
     -Will return available AdvancedDataTypes when the /types endpoint is accessed
     -Will return a AdvancedDataTypeResponse object when the /convert endpoint is accessed
     and is passed in valid arguments
-    """
+    """  # noqa: E501
 
     allow_browser_login = True
     resource_name = "advanced_data_type"
@@ -92,7 +92,7 @@ def get(self, **kwargs: Any) -> Response:
               $ref: '#/components/responses/404'
             500:
               $ref: '#/components/responses/500'
-        """
+        """  # noqa: E501
         item = kwargs["rison"]
         advanced_data_type = item["type"]
         values = item["values"]

superset/advanced_data_type/plugins/internet_address.py

@@ -65,7 +65,7 @@ def cidr_func(req: AdvancedDataTypeRequest) -> AdvancedDataTypeResponse:
             break
         else:
             resp["display_value"] = ", ".join(
-                map(
+                map(  # noqa: C417
                     lambda x: f"{x['start']} - {x['end']}"
                     if isinstance(x, dict)
                     else str(x),
@@ -76,7 +76,7 @@ def cidr_func(req: AdvancedDataTypeRequest) -> AdvancedDataTypeResponse:
 
 
 # Make this return a single clause
-def cidr_translate_filter_func(
+def cidr_translate_filter_func(  # noqa: C901
     col: Column, operator: FilterOperator, values: list[Any]
 ) -> Any:
     """

superset/advanced_data_type/plugins/internet_port.py

@@ -94,7 +94,7 @@ def port_translation_func(req: AdvancedDataTypeRequest) -> AdvancedDataTypeRespo
             break
         else:
             resp["display_value"] = ", ".join(
-                map(
+                map(  # noqa: C417
                     lambda x: f"{x['start']} - {x['end']}"
                     if isinstance(x, dict)
                     else str(x),
@@ -104,7 +104,7 @@ def port_translation_func(req: AdvancedDataTypeRequest) -> AdvancedDataTypeRespo
     return resp
 
 
-def port_translate_filter_func(
+def port_translate_filter_func(  # noqa: C901
     col: Column, operator: FilterOperator, values: list[Any]
 ) -> Any:
     """

superset/annotation_layers/annotations/api.py

@@ -181,7 +181,7 @@ def get_list(  # pylint: disable=arguments-differ
               $ref: '#/components/responses/422'
             500:
               $ref: '#/components/responses/500'
-        """
+        """  # noqa: E501
         self._apply_layered_relation_to_rison(pk, kwargs["rison"])
         return self.get_list_headless(**kwargs)