Reapply "Upgrade to Debian Bookworm and Python 3.11 (#4028)" (#4056)

This reverts commit bbf937c.

Dockerfile

@@ -1,22 +1,33 @@
 # vim: set expandtab sw=4
-FROM golang:1.22.7-bullseye
+FROM golang:1.22.7-bookworm
 
 # Create a non-priviledged user to run browsers as (Firefox and Chrome do not
 # like to run as root).
 RUN chmod a+rx $HOME && useradd --uid 9999 --user-group --create-home browser
 
+# Add apt repositories for Java, Node.js and Google Cloud CLI
+RUN export DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) && \
+    echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $DISTRO_CODENAME main" > /etc/apt/sources.list.d/adoptium.list && \
+    curl -s https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor -o /usr/share/keyrings/adoptium.gpg && \
+    export NODE_VERSION="18.x" && \
+    export ARCH=$(dpkg --print-architecture) && \
+    echo "deb [arch=$ARCH signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_VERSION nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    curl -s https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /usr/share/keyrings/nodesource.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk-$DISTRO_CODENAME main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
+
 # Sort the package names!
 # firefox-esr: provides deps for Firefox (we don't use ESR directly)
-# openjdk-11-jdk: provides JDK/JRE to Selenium & gcloud SDK
 # python-crcmod: native module to speed up CRC checksum in gsutil
+# temurin-11-jdk: provides JDK/JRE to Selenium & gcloud SDK
 RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \
         curl \
         firefox-esr \
-        lsb-release \
-        openjdk-11-jdk \
-        python3.9 \
+        nodejs \
+        python3.11 \
         python3-crcmod \
         sudo \
+        temurin-11-jdk \
         tox \
         wget \
         xvfb && \
@@ -27,21 +38,14 @@ RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \
 # discard the setting.
 RUN echo "root ALL=(ALL:ALL) ALL" > /etc/sudoers
 
-# Node LTS
-RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - && \
-    apt-get install -qqy nodejs
-
-# Google Cloud SDK
+# Google Cloud SDK configuration
 # Based on https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/Dockerfile
-RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
-    echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
-    apt-get update -qqy && apt-get install -qqy \
-        google-cloud-sdk \
-        google-cloud-sdk-app-engine-python \
-        google-cloud-sdk-app-engine-python-extras \
-        google-cloud-sdk-app-engine-go \
-        google-cloud-sdk-datastore-emulator && \
+RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \
+        google-cloud-cli \
+        google-cloud-cli-app-engine-python \
+        google-cloud-cli-app-engine-python-extras \
+        google-cloud-cli-app-engine-go \
+        google-cloud-cli-datastore-emulator && \
     gcloud config set core/disable_usage_reporting true && \
     gcloud config set component_manager/disable_update_check true && \
     gcloud --version

Makefile

@@ -229,15 +229,15 @@ gcc: apt-get-gcc
 git: apt-get-git
 jq: apt-get-jq
 psmisc: apt-get-psmisc
-python3: apt-get-python3.9
+python3: apt-get-python3.11
 tox: apt-get-tox
 unzip: apt-get-unzip
 wget: apt-get-wget
 
 java:
 	@ # java has a different apt-get package name.
 	if [[ "$$(which java)" == "" ]]; then \
-		sudo apt-get install -qqy --no-install-suggests openjdk-11-jdk; \
+		sudo apt-get install -qqy --no-install-suggests temurin-11-jdk; \
 	fi
 
 gpg:

api/query/cache/service/Dockerfile

@@ -1,6 +1,6 @@
 # Production deployment spec for query cache service.
 
-FROM golang:1.22.7-bullseye as builder
+FROM golang:1.22.7-bookworm as builder
 
 RUN apt-get update
 RUN apt-get install -qy --no-install-suggests git

docs/upgrading-go.md

@@ -15,7 +15,7 @@ Once you have confirmed that the desired version is available:
 - tooling [Dockerfile](../Dockerfile) at the root of the repo
 - searchcache [Dockerfile](../api/query/cache/service/Dockerfile)
 
-The tooling image and the first stage of searchcache use the same Golang image. Check out the Golang [page](https://hub.docker.com/_/golang?tab=tags) on Docker Hub for the latest tags. Currently, we are using the `bullseye` [release](https://wiki.debian.org/DebianReleases) of Debian. As a result pick the `golang:<latest stable version>-bullseye` tag. If bullseye is superseded by a new version, you should change that as well.
+The tooling image and the first stage of searchcache use the same Golang image. Check out the Golang [page](https://hub.docker.com/_/golang?tab=tags) on Docker Hub for the latest tags. Currently, we are using the `bookworm` [release](https://wiki.debian.org/DebianReleases) of Debian. As a result pick the `golang:<latest stable version>-bookworm` tag. If bookworm is superseded by a new version, you should change that as well.
 
 ## Step 3 - Change the version in go.mod
 

results-processor/.python-version

@@ -1 +1 @@
-3.9
+3.11

results-processor/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.20-bullseye
+FROM python:3.11.9-bookworm
 
 # Install runtime dependencies.
 # python3-crcmod for faster gsutil checksum
@@ -17,7 +17,7 @@ RUN gcloud config set disable_usage_reporting false
 RUN rm -f $HOME/.config/gcloud/gce
 
 # Setup and activate virtualenv.
-RUN virtualenv -p python3.9 /env
+RUN virtualenv -p python3.11 /env
 ENV VIRTUAL_ENV /env
 ENV PATH /env/bin:$PATH
 

results-processor/README.md

@@ -1,6 +1,6 @@
 ## Basics
 
-The results processor runs on Python 3.9. The entry point is a Flask web server
+The results processor runs on Python 3.11. The entry point is a Flask web server
 (`main.py`). In production, gunicorn is used as the WSGI (see `Dockerfile`) and
 the container runs as a custom AppEngine Flex instance (see `app.yaml`).
 
@@ -10,7 +10,7 @@ We can create a virtualenv to recreate a setup close to production for daily
 development.
 
 ```bash
-virtualenv env -p python3.9
+virtualenv env -p python3.11
 . env/bin/activate
 pip install -r requirements.txt
 ```
@@ -38,8 +38,8 @@ Dependabot is used to automatically update `requirements.txt`. To manually
 update dependencies, run the following commands:
 
 ```bash
-pip3.9 install --user pip-tools
-python3.9 -m piptools compile requirements.in
+pip3.11 install --user pip-tools
+python3.11 -m piptools compile requirements.in
 ```
 
 ## Local debugging

results-processor/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile requirements.in
@@ -53,14 +53,12 @@ grpcio==1.53.2
     # via
     #   google-api-core
     #   grpcio-status
-grpcio-status==1.44.0
+grpcio-status==1.53.2
     # via google-api-core
 gunicorn==23.0.0
     # via -r requirements.in
 idna==3.7
     # via requests
-importlib-metadata==6.8.0
-    # via flask
 itsdangerous==2.1.2
     # via flask
 jinja2==3.1.4
@@ -115,5 +113,3 @@ urllib3==2.2.3
     #   types-requests
 werkzeug==3.0.6
     # via flask
-zipp==3.19.1
-    # via importlib-metadata

results-processor/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py39
+envlist = py311
 # We don't have or need setup.py for now.
 skipsdist=True