Fix numpy and pandas versions. (#4430)

Summary:
Pull Request resolved: #4430

Numpy has to match the coremltools requirements, and Pandas depends on Numpy:
https://github.com/apple/coremltools/blob/main/reqs/build.pip

Reviewed By: kirklandsign

Differential Revision: D60265982

.ci/docker/requirements-ci.txt

@@ -1,5 +1,7 @@
 mpmath==1.3.0
-numpy==1.25.2
+numpy==1.21.3; python_version == '3.10'
+numpy==1.23.2; python_version == '3.11'
+numpy; python_version >= '3.12'
 PyYAML==6.0.1
 ruamel.yaml==0.17.32
 sympy==1.12
@@ -8,6 +10,8 @@ tomli==2.0.1
 torchsr==1.0.4
 transformers==4.38.0
 zstd==1.5.5.1
+pandas==2.0.3; python_version == '3.10'
+pandas; python_version >= '3.11'
 pytest==7.2.0
 pytest-cov==4.1.0
 expecttest==0.1.6

pyproject.toml

@@ -55,9 +55,12 @@ dependencies=[
   "flatbuffers",
   "hypothesis",
   "mpmath==1.3.0",
-  "numpy>=1.25.2",
+  "numpy==1.21.3; python_version == '3.10'",
+  "numpy==1.23.2; python_version == '3.11'",
+  "numpy; python_version >= '3.12'",
   "packaging",
-  "pandas",
+  "pandas==2.0.3; python_version == '3.10'",
+  "pandas; python_version >= '3.11'",
   "parameterized",
   "pytest",
   "pytest-xdist",