From 40db3b43099a2b87aa1a6b18f0b370f8bdb590c0 Mon Sep 17 00:00:00 2001 From: Sergey Fedoseev Date: Tue, 19 Nov 2024 12:11:02 +0100 Subject: [PATCH 1/4] Stop using S3 Select in indexer (#4212) Co-authored-by: Alexei Mochalov --- lambdas/indexer/CHANGELOG.md | 21 ++++++ lambdas/indexer/index.py | 97 ++++++++++++++++----------- lambdas/indexer/pytest.ini | 4 +- lambdas/indexer/test-requirements.txt | 1 + lambdas/indexer/test/test_index.py | 50 +------------- 5 files changed, 85 insertions(+), 88 deletions(-) create mode 100644 lambdas/indexer/CHANGELOG.md diff --git a/lambdas/indexer/CHANGELOG.md b/lambdas/indexer/CHANGELOG.md new file mode 100644 index 00000000000..c7ea99597d5 --- /dev/null +++ b/lambdas/indexer/CHANGELOG.md @@ -0,0 +1,21 @@ + +# Changelog + +Changes are listed in reverse chronological order (newer entries at the top). +The entry format is + +```markdown +- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/)) +``` + +where verb is one of + +- Removed +- Added +- Fixed +- Changed + +## Changes + +- [Changed] Stop using S3 select ([#4212](https://github.com/quiltdata/quilt/pull/4212)) +- [Added] Bootstrap the change log ([#4212](https://github.com/quiltdata/quilt/pull/4212)) diff --git a/lambdas/indexer/index.py b/lambdas/indexer/index.py index 80b6861a11f..bb6a9422229 100644 --- a/lambdas/indexer/index.py +++ b/lambdas/indexer/index.py @@ -47,6 +47,7 @@ import datetime +import functools import json import os import pathlib @@ -92,7 +93,6 @@ POINTER_PREFIX_V1, get_available_memory, get_quilt_logger, - query_manifest_content, separated_env_to_iter, ) @@ -168,12 +168,7 @@ # currently only affects .parquet, TODO: extend to other extensions assert 'SKIP_ROWS_EXTS' in os.environ SKIP_ROWS_EXTS = separated_env_to_iter('SKIP_ROWS_EXTS') -SELECT_PACKAGE_META = "SELECT * from S3Object o WHERE o.version IS NOT MISSING LIMIT 1" -# No WHERE clause needed for aggregations since S3 Select skips missing fields for aggs -SELECT_PACKAGE_STATS = ( - "SELECT COALESCE(SUM(obj['size']), 0) as total_bytes," - " COUNT(obj['size']) as total_files from S3Object obj" -) +DUCKDB_SELECT_LAMBDA_ARN = os.environ["DUCKDB_SELECT_LAMBDA_ARN"] TEST_EVENT = "s3:TestEvent" # we need to filter out GetObject and HeadObject calls generated by the present # lambda in order to display accurate analytics in the Quilt catalog @@ -182,6 +177,7 @@ logger = get_quilt_logger() +s3_client = boto3.client("s3", config=botocore.config.Config(user_agent_extra=USER_AGENT_EXTRA)) def now_like_boto3(): @@ -247,13 +243,10 @@ def select_manifest_meta(s3_client, bucket: str, key: str): wrapper for retry and returning a string """ try: - raw = query_manifest_content( - s3_client, - bucket=bucket, - key=key, - sql_stmt=SELECT_PACKAGE_META - ) - return json.load(raw) + body = s3_client.get_object(Bucket=bucket, Key=key)["Body"] + with body: # this *might* be needed to close the stream ASAP + for line in body.iter_lines(): + return json.loads(line) except (botocore.exceptions.ClientError, json.JSONDecodeError) as cle: print(f"Unable to S3 select manifest: {cle}") @@ -439,7 +432,7 @@ def get_pkg_data(): first = select_manifest_meta(s3_client, bucket, manifest_key) if not first: return - stats = select_package_stats(s3_client, bucket, manifest_key) + stats = select_package_stats(bucket, manifest_key) if not stats: return @@ -472,33 +465,54 @@ def get_pkg_data(): return True -def select_package_stats(s3_client, bucket, manifest_key) -> str: +@functools.lru_cache(maxsize=None) +def get_bucket_region(bucket: str) -> str: + resp = s3_client.head_bucket(Bucket=bucket) + return resp["ResponseMetadata"]["HTTPHeaders"]["x-amz-bucket-region"] + + +@functools.lru_cache(maxsize=None) +def get_presigner_client(bucket: str): + return boto3.client( + "s3", + region_name=get_bucket_region(bucket), + config=botocore.config.Config(signature_version="s3v4"), + ) + + +def select_package_stats(bucket, manifest_key) -> Optional[dict]: """use s3 select to generate file stats for package""" logger_ = get_quilt_logger() - try: - raw_stats = query_manifest_content( - s3_client, - bucket=bucket, - key=manifest_key, - sql_stmt=SELECT_PACKAGE_STATS - ).read() - - if raw_stats: - stats = json.loads(raw_stats) - assert isinstance(stats['total_bytes'], int) - assert isinstance(stats['total_files'], int) - - return stats - - except ( - AssertionError, - botocore.exceptions.ClientError, - json.JSONDecodeError, - KeyError, - ) as err: - logger_.exception("Unable to compute package stats via S3 select") + presigner_client = get_presigner_client(bucket) + url = presigner_client.generate_presigned_url( + ClientMethod="get_object", + Params={ + "Bucket": bucket, + "Key": manifest_key, + }, + ) + lambda_ = make_lambda_client() + q = f""" + SELECT + COALESCE(SUM(size), 0) AS total_bytes, + COUNT(size) AS total_files FROM read_ndjson('{url}', columns={{size: 'UBIGINT'}}) obj + """ + resp = lambda_.invoke( + FunctionName=DUCKDB_SELECT_LAMBDA_ARN, + Payload=json.dumps({"query": q, "user_agent": f"DuckDB Select {USER_AGENT_EXTRA}"}), + ) - return None + payload = resp["Payload"].read() + if "FunctionError" in resp: + logger_.error("DuckDB select unhandled error: %s", payload) + return None + parsed = json.loads(payload) + if "error" in parsed: + logger_.error("DuckDB select error: %s", parsed["error"]) + return None + + rows = parsed["rows"] + return rows[0] if rows else None def extract_pptx(fileobj, max_size: int) -> str: @@ -732,6 +746,11 @@ def make_s3_client(): return boto3.client("s3", config=configuration) +@functools.lru_cache(maxsize=None) +def make_lambda_client(): + return boto3.client("lambda") + + def map_event_name(event: dict): """transform eventbridge names into S3-like ones""" input_ = event["eventName"] diff --git a/lambdas/indexer/pytest.ini b/lambdas/indexer/pytest.ini index dd07825516f..f9355a4fbaf 100644 --- a/lambdas/indexer/pytest.ini +++ b/lambdas/indexer/pytest.ini @@ -1,4 +1,6 @@ [pytest] +env = + DUCKDB_SELECT_LAMBDA_ARN = "arn:aws:lambda:us-west-2:123456789012:function:select-lambda" log_cli = True # This is set above critical to prevent logger events from confusing output in CI -log_level = 51 +log_level = 51 diff --git a/lambdas/indexer/test-requirements.txt b/lambdas/indexer/test-requirements.txt index e75e43e319b..b8fc13134ea 100644 --- a/lambdas/indexer/test-requirements.txt +++ b/lambdas/indexer/test-requirements.txt @@ -5,4 +5,5 @@ pluggy==0.9 py==1.10.0 pytest==4.4.0 pytest-cov==2.6.1 +pytest-env==0.6.2 responses==0.10.14 diff --git a/lambdas/indexer/test/test_index.py b/lambdas/indexer/test/test_index.py index c53e3bfa8de..05cc0c85a1f 100644 --- a/lambdas/indexer/test/test_index.py +++ b/lambdas/indexer/test/test_index.py @@ -23,7 +23,6 @@ import responses from botocore import UNSIGNED from botocore.client import Config -from botocore.exceptions import ParamValidationError from botocore.stub import Stubber from dateutil.tz import tzutc from document_queue import EVENT_PREFIX, RetryError @@ -979,7 +978,7 @@ def test_index_if_package_select_stats_fail(self, append_mock, select_meta_mock, ) select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) - select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) + select_stats_mock.assert_called_once_with(bucket, manifest_key) append_mock.assert_called_once_with({ "_index": bucket + PACKAGE_INDEX_SUFFIX, "_id": key, @@ -1023,7 +1022,7 @@ def test_index_if_package(self, append_mock, select_meta_mock, select_stats_mock ) select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) - select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) + select_stats_mock.assert_called_once_with(bucket, manifest_key) append_mock.assert_called_once_with({ "_index": bucket + PACKAGE_INDEX_SUFFIX, "_id": key, @@ -1182,51 +1181,6 @@ def test_extension_overrides(self): assert self._get_contents('foo.txt', '.txt') == "" assert self._get_contents('foo.ipynb', '.ipynb') == "" - @pytest.mark.xfail( - raises=ParamValidationError, - reason="boto bug https://github.com/boto/botocore/issues/1621", - strict=True, - ) - def test_stub_select_object_content(self): - """Demonstrate that mocking S3 select with boto3 is broken""" - sha_hash = "50f4d0fc2c22a70893a7f356a4929046ce529b53c1ef87e28378d92b884691a5" - manifest_key = f"{MANIFEST_PREFIX_V1}{sha_hash}" - # this SHOULD work, but due to botocore bugs it does not - self.s3_stubber.add_response( - method="select_object_content", - service_response={ - "ResponseMetadata": ANY, - # it is sadly not possible to mock S3 select responses because - # boto incorrectly believes "Payload"'s value should be a dict - # but it's really an iterable in realworld code - # see https://github.com/boto/botocore/issues/1621 - "Payload": [ - { - "Stats": {} - }, - { - "Records": { - "Payload": json.dumps(MANIFEST_DATA).encode(), - }, - }, - { - "End": {} - }, - ] - }, - expected_params={ - "Bucket": "test-bucket", - "Key": manifest_key, - "Expression": index.SELECT_PACKAGE_META, - "ExpressionType": "SQL", - "InputSerialization": { - 'JSON': {'Type': 'LINES'}, - 'CompressionType': 'NONE' - }, - "OutputSerialization": {'JSON': {'RecordDelimiter': '\n'}} - } - ) - def test_synthetic_copy_event(self): """check synthetic ObjectCreated:Copy event vs organic obtained on 26-May-2020 (bucket versioning on) From c65fc4cbdf0a6be58238037c72b43dc3b83d7f21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:11:55 +0000 Subject: [PATCH 2/4] Bump aiohttp from 3.10.2 to 3.10.11 in /lambdas/tabular_preview (#4227) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lambdas/tabular_preview/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tabular_preview/requirements.txt b/lambdas/tabular_preview/requirements.txt index c787be65d4d..b10ea779083 100644 --- a/lambdas/tabular_preview/requirements.txt +++ b/lambdas/tabular_preview/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements.txt ../shared/setup.py setup.py # -aiohttp==3.10.2 +aiohttp==3.10.11 # via fsspec aiosignal==1.2.0 # via aiohttp From 8dd6815e8d9e0757e38993dae5809fb97319a13d Mon Sep 17 00:00:00 2001 From: Alexei Mochalov Date: Tue, 19 Nov 2024 17:28:43 +0500 Subject: [PATCH 3/4] Catalog: Replace S3 Select with GQL (#4218) --- catalog/CHANGELOG.md | 1 + .../app/containers/Bucket/File/Analytics.tsx | 92 ++ .../AssistantContext.ts} | 2 +- .../app/containers/Bucket/{ => File}/File.js | 90 +- .../File/gql/ObjectAccessCounts.generated.ts | 100 ++ .../File/gql/ObjectAccessCounts.graphql | 9 + catalog/app/containers/Bucket/File/index.ts | 1 + catalog/app/containers/Bucket/Overview.js | 963 ------------------ .../containers/Bucket/Overview/ColorPool.ts | 16 + .../Bucket/Overview/Downloads.spec.ts | 194 ++++ .../containers/Bucket/Overview/Downloads.tsx | 593 +++++++++++ .../app/containers/Bucket/Overview/Header.tsx | 431 ++++++++ .../Bucket/{ => Overview}/Overview-bg.jpg | Bin .../containers/Bucket/Overview/Overview.tsx | 163 +++ .../gql/BucketAccessCounts.generated.ts | 207 ++++ .../Overview/gql/BucketAccessCounts.graphql | 27 + .../gql/BucketConfig.generated.ts} | 16 +- .../gql/BucketConfig.graphql} | 0 .../app/containers/Bucket/Overview/index.tsx | 1 + catalog/app/containers/Bucket/Summarize.tsx | 6 +- .../Bucket/requests/requestsUntyped.js | 256 +---- catalog/app/embed/File.js | 75 +- catalog/app/model/graphql/schema.generated.ts | 160 ++- catalog/app/model/graphql/types.generated.ts | 29 + catalog/app/utils/AWS/S3.js | 50 +- catalog/app/utils/AWS/Signer.js | 2 +- catalog/app/utils/GraphQL/Provider.tsx | 2 + shared/graphql/schema.graphql | 13 + 28 files changed, 2114 insertions(+), 1385 deletions(-) create mode 100644 catalog/app/containers/Bucket/File/Analytics.tsx rename catalog/app/containers/Bucket/{FileAssistantContext.ts => File/AssistantContext.ts} (98%) rename catalog/app/containers/Bucket/{ => File}/File.js (85%) create mode 100644 catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.generated.ts create mode 100644 catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.graphql create mode 100644 catalog/app/containers/Bucket/File/index.ts delete mode 100644 catalog/app/containers/Bucket/Overview.js create mode 100644 catalog/app/containers/Bucket/Overview/ColorPool.ts create mode 100644 catalog/app/containers/Bucket/Overview/Downloads.spec.ts create mode 100644 catalog/app/containers/Bucket/Overview/Downloads.tsx create mode 100644 catalog/app/containers/Bucket/Overview/Header.tsx rename catalog/app/containers/Bucket/{ => Overview}/Overview-bg.jpg (100%) create mode 100644 catalog/app/containers/Bucket/Overview/Overview.tsx create mode 100644 catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.generated.ts create mode 100644 catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.graphql rename catalog/app/containers/Bucket/{OverviewBucketConfig.generated.ts => Overview/gql/BucketConfig.generated.ts} (75%) rename catalog/app/containers/Bucket/{OverviewBucketConfig.graphql => Overview/gql/BucketConfig.graphql} (100%) create mode 100644 catalog/app/containers/Bucket/Overview/index.tsx diff --git a/catalog/CHANGELOG.md b/catalog/CHANGELOG.md index 8758f53dd0a..93888fa202c 100644 --- a/catalog/CHANGELOG.md +++ b/catalog/CHANGELOG.md @@ -17,6 +17,7 @@ where verb is one of ## Changes +- [Changed] S3 Select -> GQL API calls for getting access counts ([#4218](https://github.com/quiltdata/quilt/pull/4218)) - [Changed] Athena: improve loading state and errors visuals; fix minor bugs; alphabetize and persist selection in workgroups, catalog names and databases ([#4208](https://github.com/quiltdata/quilt/pull/4208)) - [Changed] Show stack release version in footer ([#4200](https://github.com/quiltdata/quilt/pull/4200)) - [Added] Selective package downloading ([#4173](https://github.com/quiltdata/quilt/pull/4173)) diff --git a/catalog/app/containers/Bucket/File/Analytics.tsx b/catalog/app/containers/Bucket/File/Analytics.tsx new file mode 100644 index 00000000000..4152b083953 --- /dev/null +++ b/catalog/app/containers/Bucket/File/Analytics.tsx @@ -0,0 +1,92 @@ +import * as dateFns from 'date-fns' +import * as Eff from 'effect' +import * as React from 'react' +import * as M from '@material-ui/core' + +import Sparkline from 'components/Sparkline' +import * as GQL from 'utils/GraphQL' +import log from 'utils/Logging' +import * as SVG from 'utils/SVG' +import { readableQuantity } from 'utils/string' + +import Section from '../Section' + +import ACCESS_COUNTS_QUERY from './gql/ObjectAccessCounts.generated' + +const currentYear = new Date().getFullYear() + +const formatDate = (date: Date) => + dateFns.format(date, currentYear === date.getFullYear() ? 'd MMM' : 'd MMM yyyy') + +interface AnalyticsProps { + bucket: string + path: string +} + +export default function Analytics({ bucket, path }: AnalyticsProps) { + const [cursor, setCursor] = React.useState(null) + + const result = GQL.useQuery(ACCESS_COUNTS_QUERY, { bucket, key: path }) + + const data = React.useMemo(() => { + if (result.fetching) return Eff.Option.none() + if (result.error) log.error('Error fetching object access counts:', result.error) + return Eff.Option.some(Eff.Option.fromNullable(result.data?.objectAccessCounts)) + }, [result.fetching, result.error, result.data]) + + const defaultExpanded = Eff.Option.match(data, { + onNone: () => false, + onSome: Eff.Option.match({ + onNone: () => false, + onSome: ({ total }) => !!total, + }), + }) + + return ( +
+ {Eff.Option.match(data, { + onNone: () => , + onSome: Eff.Option.match({ + onNone: () => No analytics available, + onSome: ({ counts, total }) => + total ? ( + + + Downloads + + {readableQuantity(cursor === null ? total : counts[cursor].value)} + + + {cursor === null + ? `${counts.length} days` + : formatDate(counts[cursor].date)} + + + + c.value)} + onCursor={setCursor} + width={1000} + height={60} + stroke={SVG.Paint.Server( + + + + , + )} + /> + + + ) : ( + No analytics available + ), + }), + })} +
+ ) +} diff --git a/catalog/app/containers/Bucket/FileAssistantContext.ts b/catalog/app/containers/Bucket/File/AssistantContext.ts similarity index 98% rename from catalog/app/containers/Bucket/FileAssistantContext.ts rename to catalog/app/containers/Bucket/File/AssistantContext.ts index 6f5876945f8..46f06c8ccf3 100644 --- a/catalog/app/containers/Bucket/FileAssistantContext.ts +++ b/catalog/app/containers/Bucket/File/AssistantContext.ts @@ -4,7 +4,7 @@ import * as React from 'react' import * as Assistant from 'components/Assistant' import * as XML from 'utils/XML' -import { ObjectExistence } from './requests' +import { ObjectExistence } from '../requests' interface VersionsContextProps { data: $TSFixMe diff --git a/catalog/app/containers/Bucket/File.js b/catalog/app/containers/Bucket/File/File.js similarity index 85% rename from catalog/app/containers/Bucket/File.js rename to catalog/app/containers/Bucket/File/File.js index e91b4ed9fd1..fda4ceb1d40 100644 --- a/catalog/app/containers/Bucket/File.js +++ b/catalog/app/containers/Bucket/File/File.js @@ -1,6 +1,5 @@ import { basename } from 'path' -import * as dateFns from 'date-fns' import * as R from 'ramda' import * as React from 'react' import { Link, useHistory, useLocation, useParams } from 'react-router-dom' @@ -11,7 +10,6 @@ import * as Buttons from 'components/Buttons' import * as FileEditor from 'components/FileEditor' import Message from 'components/Message' import * as Preview from 'components/Preview' -import Sparkline from 'components/Sparkline' import cfg from 'constants/config' import * as Bookmarks from 'containers/Bookmarks' import * as Notifications from 'containers/Notifications' @@ -21,23 +19,24 @@ import * as BucketPreferences from 'utils/BucketPreferences' import { useData } from 'utils/Data' import MetaTitle from 'utils/MetaTitle' import * as NamedRoutes from 'utils/NamedRoutes' -import * as SVG from 'utils/SVG' import { linkStyle } from 'utils/StyledLink' import copyToClipboard from 'utils/clipboard' import * as Format from 'utils/format' import parseSearch from 'utils/parseSearch' import { up, decode, handleToHttpsUri } from 'utils/s3paths' -import { readableBytes, readableQuantity } from 'utils/string' - -import AssistButton from './AssistButton' -import FileCodeSamples from './CodeSamples/File' -import * as AssistantContext from './FileAssistantContext' -import FileProperties from './FileProperties' -import * as FileView from './FileView' -import Section from './Section' -import renderPreview from './renderPreview' -import * as requests from './requests' -import { useViewModes, viewModeToSelectOption } from './viewModes' +import { readableBytes } from 'utils/string' + +import AssistButton from '../AssistButton' +import FileCodeSamples from '../CodeSamples/File' +import FileProperties from '../FileProperties' +import * as FileView from '../FileView' +import Section from '../Section' +import renderPreview from '../renderPreview' +import * as requests from '../requests' +import { useViewModes, viewModeToSelectOption } from '../viewModes' + +import Analytics from './Analytics' +import * as AssistantContext from './AssistantContext' const useVersionInfoStyles = M.makeStyles(({ typography }) => ({ version: { @@ -203,69 +202,6 @@ function VersionInfo({ bucket, path, version }) { ) } -function Analytics({ bucket, path }) { - const [cursor, setCursor] = React.useState(null) - const s3 = AWS.S3.use() - const today = React.useMemo(() => new Date(), []) - const formatDate = (date) => - dateFns.format( - date, - today.getFullYear() === date.getFullYear() ? 'd MMM' : 'd MMM yyyy', - ) - const data = useData(requests.objectAccessCounts, { s3, bucket, path, today }) - - const defaultExpanded = data.case({ - Ok: ({ total }) => !!total, - _: () => false, - }) - - return ( -
- {data.case({ - Ok: ({ counts, total }) => - total ? ( - - - Downloads - - {readableQuantity(cursor === null ? total : counts[cursor].value)} - - - {cursor === null - ? `${counts.length} days` - : formatDate(counts[cursor].date)} - - - - - - - , - )} - /> - - - ) : ( - No analytics available - ), - Err: () => No analytics available, - _: () => , - })} -
- ) -} - function CenteredProgress() { return ( diff --git a/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.generated.ts b/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.generated.ts new file mode 100644 index 00000000000..94875020cfe --- /dev/null +++ b/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.generated.ts @@ -0,0 +1,100 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core' +import * as Types from '../../../../model/graphql/types.generated' + +export type containers_Bucket_File_gql_ObjectAccessCountsQueryVariables = Types.Exact<{ + bucket: Types.Scalars['String'] + key: Types.Scalars['String'] +}> + +export type containers_Bucket_File_gql_ObjectAccessCountsQuery = { + readonly __typename: 'Query' +} & { + readonly objectAccessCounts: Types.Maybe< + { readonly __typename: 'AccessCounts' } & Pick & { + readonly counts: ReadonlyArray< + { readonly __typename: 'AccessCountForDate' } & Pick< + Types.AccessCountForDate, + 'date' | 'value' + > + > + } + > +} + +export const containers_Bucket_File_gql_ObjectAccessCountsDocument = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'containers_Bucket_File_gql_ObjectAccessCounts' }, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: { kind: 'Variable', name: { kind: 'Name', value: 'bucket' } }, + type: { + kind: 'NonNullType', + type: { kind: 'NamedType', name: { kind: 'Name', value: 'String' } }, + }, + }, + { + kind: 'VariableDefinition', + variable: { kind: 'Variable', name: { kind: 'Name', value: 'key' } }, + type: { + kind: 'NonNullType', + type: { kind: 'NamedType', name: { kind: 'Name', value: 'String' } }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'objectAccessCounts' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'bucket' }, + value: { kind: 'Variable', name: { kind: 'Name', value: 'bucket' } }, + }, + { + kind: 'Argument', + name: { kind: 'Name', value: 'key' }, + value: { kind: 'Variable', name: { kind: 'Name', value: 'key' } }, + }, + { + kind: 'Argument', + name: { kind: 'Name', value: 'window' }, + value: { kind: 'IntValue', value: '365' }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'total' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'counts' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'date' } }, + { kind: 'Field', name: { kind: 'Name', value: 'value' } }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode< + containers_Bucket_File_gql_ObjectAccessCountsQuery, + containers_Bucket_File_gql_ObjectAccessCountsQueryVariables +> + +export { containers_Bucket_File_gql_ObjectAccessCountsDocument as default } diff --git a/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.graphql b/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.graphql new file mode 100644 index 00000000000..431f1cb2ee2 --- /dev/null +++ b/catalog/app/containers/Bucket/File/gql/ObjectAccessCounts.graphql @@ -0,0 +1,9 @@ +query ($bucket: String!, $key: String!) { + objectAccessCounts(bucket: $bucket, key: $key, window: 365) { + total + counts { + date + value + } + } +} diff --git a/catalog/app/containers/Bucket/File/index.ts b/catalog/app/containers/Bucket/File/index.ts new file mode 100644 index 00000000000..d1590f6b882 --- /dev/null +++ b/catalog/app/containers/Bucket/File/index.ts @@ -0,0 +1 @@ +export { default } from './File' diff --git a/catalog/app/containers/Bucket/Overview.js b/catalog/app/containers/Bucket/Overview.js deleted file mode 100644 index 3acef4e2ccb..00000000000 --- a/catalog/app/containers/Bucket/Overview.js +++ /dev/null @@ -1,963 +0,0 @@ -import cx from 'classnames' -import * as dateFns from 'date-fns' -import * as R from 'ramda' -import * as React from 'react' -import { Link as RRLink, useParams } from 'react-router-dom' -import * as redux from 'react-redux' -import * as M from '@material-ui/core' -import { fade } from '@material-ui/core/styles' -import useComponentSize from '@rehooks/component-size' - -import Skeleton from 'components/Skeleton' -import StackedAreaChart from 'components/StackedAreaChart' -import cfg from 'constants/config' -import * as authSelectors from 'containers/Auth/selectors' -import * as APIConnector from 'utils/APIConnector' -import * as AWS from 'utils/AWS' -import AsyncResult from 'utils/AsyncResult' -import * as BucketPreferences from 'utils/BucketPreferences' -import Data, { useData } from 'utils/Data' -import { useQueryS } from 'utils/GraphQL' -import * as LinkedData from 'utils/LinkedData' -import * as NamedRoutes from 'utils/NamedRoutes' -import * as SVG from 'utils/SVG' -import { readableBytes, readableQuantity, formatQuantity } from 'utils/string' - -import * as Gallery from './Gallery' -import * as Summarize from './Summarize' -import * as requests from './requests' -import BUCKET_CONFIG_QUERY from './OverviewBucketConfig.generated' - -import bg from './Overview-bg.jpg' - -const RODA_LINK = 'https://registry.opendata.aws' -const RODA_BUCKET = 'quilt-open-data-bucket' -const MAX_EXTS = 7 -// must have length >= MAX_EXTS -const COLOR_MAP = [ - '#8ad3cb', - '#d7ce69', - '#bfbadb', - '#f4806c', - '#83b0d1', - '#b2de67', - '#bc81be', - '#f0b5d3', - '#7ba39f', - '#9894ad', - '#be7265', - '#94ad6b', -] - -function mkKeyedPool(pool) { - const map = {} - let poolIdx = 0 - const get = (key) => { - if (!(key in map)) { - // eslint-disable-next-line no-plusplus - map[key] = pool[poolIdx++ % pool.length] - } - return map[key] - } - return { get } -} - -function useConst(cons) { - const ref = React.useRef(null) - if (!ref.current) ref.current = { value: cons() } - return ref.current.value -} - -const useObjectsByExtStyles = M.makeStyles((t) => ({ - root: { - display: 'grid', - gridAutoRows: 20, - gridColumnGap: t.spacing(1), - gridRowGap: t.spacing(0.25), - gridTemplateAreas: ` - ". heading heading" - `, - gridTemplateColumns: 'minmax(30px, max-content) 1fr minmax(30px, max-content)', - gridTemplateRows: 'auto', - [t.breakpoints.down('sm')]: { - gridTemplateAreas: ` - "heading heading heading" - `, - }, - }, - heading: { - ...t.typography.h6, - gridArea: 'heading', - marginBottom: t.spacing(1), - [t.breakpoints.down('sm')]: { - textAlign: 'center', - }, - }, - ext: { - color: t.palette.text.secondary, - gridColumn: 1, - fontSize: t.typography.overline.fontSize, - fontWeight: t.typography.fontWeightMedium, - letterSpacing: t.typography.subtitle2.letterSpacing, - lineHeight: t.typography.pxToRem(20), - textAlign: 'right', - }, - count: { - color: t.palette.text.secondary, - gridColumn: 3, - fontSize: t.typography.overline.fontSize, - fontWeight: t.typography.fontWeightMedium, - letterSpacing: t.typography.subtitle2.letterSpacing, - lineHeight: t.typography.pxToRem(20), - }, - bar: { - background: t.palette.action.hover, - gridColumn: 2, - }, - gauge: { - height: '100%', - position: 'relative', - }, - flip: {}, - size: { - color: t.palette.common.white, - fontSize: t.typography.overline.fontSize, - fontWeight: t.typography.fontWeightMedium, - letterSpacing: t.typography.subtitle2.letterSpacing, - lineHeight: t.typography.pxToRem(20), - position: 'absolute', - right: t.spacing(1), - '&$flip': { - color: t.palette.text.hint, - left: `calc(100% + ${t.spacing(1)}px)`, - right: 'auto', - }, - }, - skeleton: { - gridColumn: '1 / span 3', - }, - unavail: { - ...t.typography.body2, - alignItems: 'center', - display: 'flex', - gridColumn: '1 / span 3', - gridRow: `2 / span ${MAX_EXTS}`, - justifyContent: 'center', - }, -})) - -function ObjectsByExt({ data, colorPool, ...props }) { - const classes = useObjectsByExtStyles() - return ( - -
Objects by File Extension
- {AsyncResult.case( - { - Ok: (exts) => { - const capped = exts.slice(0, MAX_EXTS) - const maxBytes = capped.reduce((max, e) => Math.max(max, e.bytes), 0) - const max = Math.log(maxBytes + 1) - const scale = (x) => Math.log(x + 1) / max - return capped.map(({ ext, bytes, objects }, i) => { - const color = colorPool.get(ext) - return ( - -
- {ext || 'other'} -
-
-
-
- {readableBytes(bytes)} -
-
-
-
- {readableQuantity(objects)} -
- - ) - }) - }, - _: (r) => ( - <> - {R.times( - (i) => ( - - ), - MAX_EXTS, - )} - {AsyncResult.Err.is(r) && ( -
Data unavailable
- )} - - ), - }, - data, - )} - - ) -} - -const skelData = R.times( - R.pipe( - () => R.times(Math.random, 30), - R.scan(R.add, 0), - R.drop(1), - R.map((v) => Math.log(100 * v + 1)), - ), - 8, -) - -const skelColors = [ - [M.colors.grey[300], M.colors.grey[100]], - [M.colors.grey[400], M.colors.grey[200]], -] - -const mkPulsingGradient = ({ colors: [c1, c2], animate = false }) => - SVG.Paint.Server( - - - {animate && ( - - )} - - , - ) - -function ChartSkel({ - height, - width, - lines = skelData.length, - animate = false, - children, -}) { - const data = React.useMemo( - () => R.times((i) => skelData[i % skelData.length], lines), - [lines], - ) - const fills = React.useMemo( - () => - R.times( - (i) => mkPulsingGradient({ colors: skelColors[i % skelColors.length], animate }), - lines, - ), - [lines, animate], - ) - return ( - - - {children} - - ) -} - -const ANALYTICS_WINDOW_OPTIONS = [ - { value: 31, label: 'Last 1 month' }, - { value: 91, label: 'Last 3 months' }, - { value: 182, label: 'Last 6 months' }, - { value: 365, label: 'Last 12 months' }, -] - -function DownloadsRange({ value, onChange, bucket, rawData }) { - const [anchor, setAnchor] = React.useState(null) - - const open = React.useCallback( - (e) => { - setAnchor(e.target) - }, - [setAnchor], - ) - - const close = React.useCallback(() => { - setAnchor(null) - }, [setAnchor]) - - const choose = React.useCallback( - (e) => { - onChange(e.target.value) - close() - }, - [onChange, close], - ) - - const { label } = ANALYTICS_WINDOW_OPTIONS.find((o) => o.value === value) || {} - - return ( - <> - - - {label} expand_more - - - {ANALYTICS_WINDOW_OPTIONS.map((o) => ( - - {o.label} - - ))} - - - Download to file - - - - ) -} - -const useStatsTipStyles = M.makeStyles((t) => ({ - root: { - background: fade(t.palette.grey[700], 0.9), - color: t.palette.common.white, - padding: [[6, 8]], - }, - head: { - display: 'flex', - justifyContent: 'space-between', - marginBottom: 4, - }, - date: {}, - total: {}, - extsContainer: { - alignItems: 'center', - display: 'grid', - gridAutoRows: 'auto', - gridColumnGap: 4, - gridTemplateColumns: 'max-content max-content 1fr', - }, - ext: { - fontSize: 12, - lineHeight: '16px', - maxWidth: 70, - opacity: 0.6, - overflow: 'hidden', - textAlign: 'right', - textOverflow: 'ellipsis', - }, - color: { - borderRadius: '50%', - height: 8, - opacity: 0.6, - width: 8, - }, - number: { - fontSize: 12, - lineHeight: '16px', - opacity: 0.6, - }, - hl: { - opacity: 1, - }, -})) - -function StatsTip({ stats, colorPool, className, ...props }) { - const classes = useStatsTipStyles() - return ( - -
-
{dateFns.format(stats.date, 'd MMM')}
-
- {readableQuantity(stats.combined.sum)} (+ - {readableQuantity(stats.combined.value)}) -
-
-
- {stats.byExt.map((s) => { - const hl = stats.highlighted ? stats.highlighted.ext === s.ext : true - return ( - -
{s.ext || 'other'}
-
-
- {readableQuantity(s.sum)} (+ - {readableQuantity(s.value)}) -
- - ) - })} -
- - ) -} - -const Transition = ({ TransitionComponent = M.Grow, children, ...props }) => { - const contentsRef = React.useRef(null) - if (props.in) contentsRef.current = children() - return ( - contentsRef.current && ( - {contentsRef.current} - ) - ) -} - -// use the same height as the bar chart: 20px per bar with 2px margin -const CHART_H = 22 * MAX_EXTS - 2 - -const useDownloadsStyles = M.makeStyles((t) => ({ - root: { - display: 'grid', - gridRowGap: t.spacing(0.25), - gridTemplateAreas: ` - "heading period" - "chart chart" - `, - gridTemplateColumns: 'min-content 1fr', - gridTemplateRows: 'auto auto', - [t.breakpoints.down('sm')]: { - gridTemplateAreas: ` - "heading" - "chart" - "period" - `, - gridTemplateColumns: '1fr', - gridTemplateRows: 'auto auto auto', - }, - }, - heading: { - ...t.typography.h6, - gridArea: 'heading', - marginBottom: t.spacing(1), - whiteSpace: 'nowrap', - [t.breakpoints.down('sm')]: { - marginBottom: 0, - textAlign: 'center', - }, - }, - ext: { - display: 'inline-block', - maxWidth: 100, - overflow: 'hidden', - textOverflow: 'ellipsis', - verticalAlign: 'bottom', - }, - period: { - display: 'flex', - gridArea: 'period', - justifyContent: 'center', - alignItems: 'center', - [t.breakpoints.down('sm')]: { - paddingBottom: t.spacing(1), - paddingTop: t.spacing(2), - }, - [t.breakpoints.up('md')]: { - height: 37, - justifyContent: 'flex-end', - }, - }, - chart: { - gridArea: 'chart', - position: 'relative', - }, - left: {}, - right: {}, - dateStats: { - maxWidth: 180, - position: 'absolute', - top: 0, - width: 'calc(50% - 8px)', - zIndex: 1, - '&$left': { - left: 0, - }, - '&$right': { - right: 0, - }, - }, - unavail: { - ...t.typography.body2, - alignItems: 'center', - display: 'flex', - height: '100%', - justifyContent: 'center', - position: 'absolute', - top: 0, - width: '100%', - }, -})) - -function Downloads({ bucket, colorPool, ...props }) { - const s3 = AWS.S3.use() - const today = React.useMemo(() => new Date(), []) - const classes = useDownloadsStyles() - const ref = React.useRef(null) - const { width } = useComponentSize(ref) - const [window, setWindow] = React.useState(ANALYTICS_WINDOW_OPTIONS[0].value) - const [cursor, setCursor] = React.useState(null) - const cursorStats = (counts) => { - if (!cursor) return null - const { date, ...combined } = counts.combined.counts[cursor.j] - const byExt = counts.byExtCollapsed.map((e) => ({ - ext: e.ext, - ...e.counts[cursor.j], - })) - const highlighted = cursor.i == null ? null : counts.byExtCollapsed[cursor.i] - const firstHalf = cursor.j < counts.combined.counts.length / 2 - return { date, combined, byExt, highlighted, firstHalf } - } - - const mkRawData = AsyncResult.case({ - Ok: (data) => `data:application/json,${JSON.stringify(data)}`, - _: () => null, - }) - - if (!cfg.analyticsBucket) { - return ( - -
Requires CloudTrail
- - ) - } - - return ( - - {(data) => ( - -
- -
-
- {AsyncResult.case( - { - Ok: (counts) => { - const stats = cursorStats(counts) - const hl = stats && stats.highlighted - const ext = hl ? hl.ext || 'other' : 'total' - const total = hl ? hl.total : counts.combined.total - if (!counts.byExtCollapsed.length) return 'Downloads' - return ( - <> - Downloads ({ext}):{' '} - {readableQuantity(total)} - - ) - }, - _: () => 'Downloads', - }, - data, - )} -
-
- {AsyncResult.case( - { - Ok: (counts) => { - if (!counts.byExtCollapsed.length) { - return ( - -
No Data
- - ) - } - - const stats = cursorStats(counts) - return ( - <> - - e.counts.map((i) => Math.log(i.sum + 1)), - )} - onCursor={setCursor} - height={CHART_H} - width={width} - areaFills={counts.byExtCollapsed.map((e) => - SVG.Paint.Color(colorPool.get(e.ext)), - )} - lineStroke={SVG.Paint.Color(M.colors.grey[500])} - extendL - extendR - px={10} - /> - - {() => ( - - )} - - - {() => ( - - )} - - - ) - }, - _: () => , - }, - data, - )} -
- - )} - - ) -} - -const useStatDisplayStyles = M.makeStyles((t) => ({ - root: { - alignItems: 'baseline', - display: 'flex', - '& + &': { - marginLeft: t.spacing(1.5), - [t.breakpoints.up('sm')]: { - marginLeft: t.spacing(4), - }, - [t.breakpoints.up('md')]: { - marginLeft: t.spacing(6), - }, - }, - }, - value: { - fontSize: t.typography.h6.fontSize, - fontWeight: t.typography.fontWeightBold, - letterSpacing: 0, - lineHeight: '20px', - [t.breakpoints.up('sm')]: { - fontSize: t.typography.h4.fontSize, - lineHeight: '32px', - }, - }, - label: { - ...t.typography.body2, - color: t.palette.grey[300], - lineHeight: 1, - marginLeft: t.spacing(0.5), - [t.breakpoints.up('sm')]: { - marginLeft: t.spacing(1), - }, - }, - skeletonContainer: { - alignItems: 'center', - height: 20, - [t.breakpoints.up('sm')]: { - height: 32, - }, - }, - skeleton: { - borderRadius: t.shape.borderRadius, - height: t.typography.h6.fontSize, - width: 96, - [t.breakpoints.up('sm')]: { - height: t.typography.h4.fontSize, - width: 120, - }, - }, -})) - -function StatDisplay({ value, label, format, fallback }) { - const classes = useStatDisplayStyles() - return R.pipe( - AsyncResult.case({ - Ok: R.pipe(format || R.identity, AsyncResult.Ok), - Err: R.pipe(fallback || R.identity, AsyncResult.Ok), - _: R.identity, - }), - AsyncResult.case({ - Ok: (v) => - v != null && ( - - {v} - {!!label && {label}} - - ), - _: () => ( -
- -
- ), - }), - )(value) -} - -const useHeadStyles = M.makeStyles((t) => ({ - root: { - position: 'relative', - [t.breakpoints.down('xs')]: { - borderRadius: 0, - }, - [t.breakpoints.up('sm')]: { - marginTop: t.spacing(2), - }, - }, - top: { - background: `center / cover url(${bg}) ${t.palette.grey[700]}`, - borderTopLeftRadius: t.shape.borderRadius, - borderTopRightRadius: t.shape.borderRadius, - color: t.palette.common.white, - overflow: 'hidden', - paddingBottom: t.spacing(3), - paddingLeft: t.spacing(2), - paddingRight: t.spacing(2), - paddingTop: t.spacing(4), - position: 'relative', - [t.breakpoints.up('sm')]: { - padding: t.spacing(4), - }, - [t.breakpoints.down('xs')]: { - borderRadius: 0, - }, - }, - settings: { - color: t.palette.common.white, - position: 'absolute', - right: t.spacing(2), - top: t.spacing(2), - }, -})) - -function Head({ s3, overviewUrl, bucket, description }) { - const classes = useHeadStyles() - const req = APIConnector.use() - const isRODA = !!overviewUrl && overviewUrl.includes(`/${RODA_BUCKET}/`) - const colorPool = useConst(() => mkKeyedPool(COLOR_MAP)) - const statsData = useData(requests.bucketStats, { req, s3, bucket, overviewUrl }) - const pkgCountData = useData(requests.countPackageRevisions, { req, bucket }) - const { urls } = NamedRoutes.use() - const isAdmin = redux.useSelector(authSelectors.isAdmin) - return ( - - - {bucket} - {!!description && ( - - {description} - - )} - {isRODA && ( - - - From the{' '} - - Registry of Open Data on AWS - - - - )} - - '? B'} - /> - '?'} - /> - null} - /> - - {isAdmin && ( - - - settings - - - )} - - - - - - - - - - - - ) -} - -function Readmes({ s3, overviewUrl, bucket }) { - return ( - - {AsyncResult.case({ - Ok: (rs) => - (rs.discovered.length > 0 || !!rs.forced) && ( - <> - {!!rs.forced && ( - - )} - {rs.discovered.map((h) => ( - - ))} - - ), - _: () => , - })} - - ) -} - -function Imgs({ s3, overviewUrl, inStack, bucket }) { - const req = APIConnector.use() - return ( - - {AsyncResult.case({ - Ok: (images) => (images.length ? : null), - _: () => , - })} - - ) -} - -function ThumbnailsWrapper({ - s3, - overviewUrl, - inStack, - bucket, - preferences: galleryPrefs, -}) { - if (cfg.noOverviewImages || !galleryPrefs) return null - if (!galleryPrefs.overview) return null - return ( - - {AsyncResult.case({ - Ok: (h) => - (!h || galleryPrefs.summarize) && ( - - ), - Err: () => , - Pending: () => , - _: () => null, - })} - - ) -} - -export default function Overview() { - const { bucket } = useParams() - - const s3 = AWS.S3.use() - const { bucketConfig } = useQueryS(BUCKET_CONFIG_QUERY, { bucket }) - const inStack = !!bucketConfig - const overviewUrl = bucketConfig?.overviewUrl - const description = bucketConfig?.description - const prefs = BucketPreferences.use() - return ( - - {inStack && ( - - - - )} - {bucketConfig ? ( - - ) : ( - - {bucket} - - )} - - {BucketPreferences.Result.match( - { - Ok: ({ ui: { blocks } }) => ( - - ), - Pending: () => , - Init: R.F, - }, - prefs, - )} - - - ) -} diff --git a/catalog/app/containers/Bucket/Overview/ColorPool.ts b/catalog/app/containers/Bucket/Overview/ColorPool.ts new file mode 100644 index 00000000000..6be8be62756 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/ColorPool.ts @@ -0,0 +1,16 @@ +export interface ColorPool { + get: (key: string) => string +} + +export function makeColorPool(pool: string[]): ColorPool { + const map: Record = {} + let poolIdx = 0 + const get = (key: string): string => { + if (!(key in map)) { + // eslint-disable-next-line no-plusplus + map[key] = pool[poolIdx++ % pool.length] + } + return map[key] + } + return { get } +} diff --git a/catalog/app/containers/Bucket/Overview/Downloads.spec.ts b/catalog/app/containers/Bucket/Overview/Downloads.spec.ts new file mode 100644 index 00000000000..0611d0b7b54 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/Downloads.spec.ts @@ -0,0 +1,194 @@ +import { processBucketAccessCounts } from './Downloads' + +jest.mock( + 'constants/config', + jest.fn(() => ({})), +) + +describe('containers/Bucket/Overview/Downloads', () => { + describe('processBucketAccessCounts', () => { + it('should normalize the data received from GQL and compute some missing data', () => { + expect( + processBucketAccessCounts({ + __typename: 'BucketAccessCounts', + byExt: [ + { + __typename: 'AccessCountsGroup', + ext: 'csv', + counts: { + __typename: 'AccessCounts', + total: 10, + counts: [ + { + __typename: 'AccessCountForDate', + value: 1, + date: new Date('2021-08-01'), + }, + { + __typename: 'AccessCountForDate', + value: 2, + date: new Date('2021-08-02'), + }, + { + __typename: 'AccessCountForDate', + value: 3, + date: new Date('2021-08-03'), + }, + { + __typename: 'AccessCountForDate', + value: 4, + date: new Date('2021-08-04'), + }, + ], + }, + }, + ], + byExtCollapsed: [ + { + __typename: 'AccessCountsGroup', + ext: 'csv', + counts: { + __typename: 'AccessCounts', + total: 10, + counts: [ + { + __typename: 'AccessCountForDate', + value: 1, + date: new Date('2021-08-01'), + }, + { + __typename: 'AccessCountForDate', + value: 2, + date: new Date('2021-08-02'), + }, + { + __typename: 'AccessCountForDate', + value: 3, + date: new Date('2021-08-03'), + }, + { + __typename: 'AccessCountForDate', + value: 4, + date: new Date('2021-08-04'), + }, + ], + }, + }, + ], + combined: { + __typename: 'AccessCounts', + total: 10, + counts: [ + { + __typename: 'AccessCountForDate', + value: 1, + date: new Date('2021-08-01'), + }, + { + __typename: 'AccessCountForDate', + value: 2, + date: new Date('2021-08-02'), + }, + { + __typename: 'AccessCountForDate', + value: 3, + date: new Date('2021-08-03'), + }, + { + __typename: 'AccessCountForDate', + value: 4, + date: new Date('2021-08-04'), + }, + ], + }, + }), + ).toEqual({ + byExt: [ + { + ext: '.csv', + counts: { + total: 10, + counts: [ + { + date: new Date('2021-08-01'), + value: 1, + sum: 1, + }, + { + date: new Date('2021-08-02'), + value: 2, + sum: 3, + }, + { + date: new Date('2021-08-03'), + value: 3, + sum: 6, + }, + { + date: new Date('2021-08-04'), + value: 4, + sum: 10, + }, + ], + }, + }, + ], + byExtCollapsed: [ + { + ext: '.csv', + counts: { + total: 10, + counts: [ + { + date: new Date('2021-08-01'), + value: 1, + sum: 1, + }, + { + date: new Date('2021-08-02'), + value: 2, + sum: 3, + }, + { + date: new Date('2021-08-03'), + value: 3, + sum: 6, + }, + { + date: new Date('2021-08-04'), + value: 4, + sum: 10, + }, + ], + }, + }, + ], + combined: { + total: 10, + counts: [ + { + date: new Date('2021-08-01'), + value: 1, + sum: 1, + }, + { + date: new Date('2021-08-02'), + value: 2, + sum: 3, + }, + { + date: new Date('2021-08-03'), + value: 3, + sum: 6, + }, + { + date: new Date('2021-08-04'), + value: 4, + sum: 10, + }, + ], + }, + }) + }) + }) +}) diff --git a/catalog/app/containers/Bucket/Overview/Downloads.tsx b/catalog/app/containers/Bucket/Overview/Downloads.tsx new file mode 100644 index 00000000000..065519d45b2 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/Downloads.tsx @@ -0,0 +1,593 @@ +import cx from 'classnames' +import * as dateFns from 'date-fns' +import * as Eff from 'effect' +import * as React from 'react' +import * as M from '@material-ui/core' +import { fade } from '@material-ui/core/styles' +import useComponentSize from '@rehooks/component-size' + +import StackedAreaChart from 'components/StackedAreaChart' +import cfg from 'constants/config' +import * as GQL from 'utils/GraphQL' +import log from 'utils/Logging' +import * as SVG from 'utils/SVG' +import { readableQuantity } from 'utils/string' + +import { ColorPool } from './ColorPool' + +import BUCKET_ACCESS_COUNTS_QUERY from './gql/BucketAccessCounts.generated' + +type GQLBucketAccessCounts = NonNullable< + GQL.DataForDoc['bucketAccessCounts'] +> +type GQLAccessCountsGroup = GQLBucketAccessCounts['byExt'][0] +type GQLAccessCounts = GQLBucketAccessCounts['combined'] +type GQLAccessCountForDate = GQLAccessCounts['counts'][0] + +interface ProcessedAccessCountForDate { + date: Date + value: number + sum: number +} + +interface ProcessedAccessCounts { + total: number + counts: readonly ProcessedAccessCountForDate[] +} + +interface ProcessedAccessCountsGroup { + ext: string + counts: ProcessedAccessCounts +} + +interface ProcessedBucketAccessCounts { + byExt: readonly ProcessedAccessCountsGroup[] + byExtCollapsed: readonly ProcessedAccessCountsGroup[] + combined: ProcessedAccessCounts +} + +const processAccessCountForDateArr = ( + counts: readonly GQLAccessCountForDate[], +): readonly ProcessedAccessCountForDate[] => + // compute running sum + Eff.Array.mapAccum(counts, 0, (acc, { value, date }) => [ + acc + value, + { + value, + date, + sum: acc + value, + }, + ])[1] + +const processAccessCounts = (counts: GQLAccessCounts): ProcessedAccessCounts => ({ + total: counts.total, + counts: processAccessCountForDateArr(counts.counts), +}) + +const processAccessCountsGroup = ( + group: GQLAccessCountsGroup, +): ProcessedAccessCountsGroup => ({ + ext: group.ext && `.${group.ext}`, + counts: processAccessCounts(group.counts), +}) + +export const processBucketAccessCounts = ( + counts: GQLBucketAccessCounts, +): ProcessedBucketAccessCounts => ({ + byExt: counts.byExt.map(processAccessCountsGroup), + byExtCollapsed: counts.byExtCollapsed.map(processAccessCountsGroup), + combined: processAccessCounts(counts.combined), +}) + +interface Cursor { + i: number | null // ext + j: number // date +} + +interface CursorStats { + date: Date + combined: { + sum: number + value: number + } + byExt: { + ext: string + sum: number + value: number + date: Date + }[] + highlighted: { + ext: string + counts: ProcessedAccessCounts + } | null + firstHalf: boolean +} + +function getCursorStats( + counts: ProcessedBucketAccessCounts, + cursor: Cursor | null, +): CursorStats | null { + if (!cursor) return null + + const { date, ...combined } = counts.combined.counts[cursor.j] + const byExt = counts.byExtCollapsed.map((e) => ({ + ext: e.ext, + ...e.counts.counts[cursor.j], + })) + const highlighted = cursor.i == null ? null : counts.byExtCollapsed[cursor.i] + const firstHalf = cursor.j < counts.combined.counts.length / 2 + return { date, combined, byExt, highlighted, firstHalf } +} + +const skelData = Eff.Array.makeBy( + 8, + Eff.flow( + () => Eff.Array.makeBy(30, Math.random), + Eff.Array.scan(0, Eff.Number.sum), + Eff.Array.drop(1), + Eff.Array.map((v) => Math.log(100 * v + 1)), + ), +) + +const skelColors = [ + [M.colors.grey[300], M.colors.grey[100]], + [M.colors.grey[400], M.colors.grey[200]], +] as const + +const mkPulsingGradient = ([c1, c2]: readonly [string, string], animate: boolean) => + SVG.Paint.Server( + + + {animate && ( + + )} + + , + ) + +interface ChartSkelProps { + height: number + width: number + lines?: number + animate?: boolean + children?: React.ReactNode +} + +function ChartSkel({ + height, + width, + lines = skelData.length, + animate = false, + children, +}: ChartSkelProps) { + const data = React.useMemo( + () => Eff.Array.makeBy(lines, (i) => skelData[i % skelData.length]), + [lines], + ) + const fills = React.useMemo( + () => + Eff.Array.makeBy(lines, (i) => + mkPulsingGradient(skelColors[i % skelColors.length], animate), + ), + [lines, animate], + ) + return ( + + {/* @ts-expect-error */} + + {children} + + ) +} + +const ANALYTICS_WINDOW_OPTIONS = [ + { value: 31, label: 'Last 1 month' }, + { value: 91, label: 'Last 3 months' }, + { value: 182, label: 'Last 6 months' }, + { value: 365, label: 'Last 12 months' }, +] + +interface DownloadsRangeProps { + value: number + onChange: (value: number) => void + bucket: string + data: Eff.Option.Option +} + +function DownloadsRange({ value, onChange, bucket, data }: DownloadsRangeProps) { + const [anchor, setAnchor] = React.useState(null) + + const open = React.useCallback( + (e) => { + setAnchor(e.target) + }, + [setAnchor], + ) + + const close = React.useCallback(() => { + setAnchor(null) + }, [setAnchor]) + + const choose = React.useCallback( + (e) => { + onChange(e.target.value) + close() + }, + [onChange, close], + ) + + const { label } = ANALYTICS_WINDOW_OPTIONS.find((o) => o.value === value) || {} + + const jsonData = React.useMemo( + () => + Eff.Option.match(data, { + onNone: () => null, + onSome: (d) => `data:application/json,${JSON.stringify(d)}`, + }), + [data], + ) + + return ( + <> + + + {label} expand_more + + + {ANALYTICS_WINDOW_OPTIONS.map((o) => ( + + {o.label} + + ))} + + + Download to file + + + + ) +} + +const useStatsTipStyles = M.makeStyles((t) => ({ + root: { + background: fade(t.palette.grey[700], 0.9), + color: t.palette.common.white, + padding: '6px 8px', + }, + head: { + display: 'flex', + justifyContent: 'space-between', + marginBottom: 4, + }, + date: {}, + total: {}, + extsContainer: { + alignItems: 'center', + display: 'grid', + gridAutoRows: 'auto', + gridColumnGap: 4, + gridTemplateColumns: 'max-content max-content 1fr', + }, + ext: { + fontSize: 12, + lineHeight: '16px', + maxWidth: 70, + opacity: 0.6, + overflow: 'hidden', + textAlign: 'right', + textOverflow: 'ellipsis', + }, + color: { + borderRadius: '50%', + height: 8, + opacity: 0.6, + width: 8, + }, + number: { + fontSize: 12, + lineHeight: '16px', + opacity: 0.6, + }, + hl: { + opacity: 1, + }, +})) + +interface StatsTipProps { + stats: CursorStats | null + colorPool: ColorPool + className?: string +} + +function StatsTip({ stats, colorPool, className, ...props }: StatsTipProps) { + const classes = useStatsTipStyles() + if (!stats) return null + return ( + +
+
{dateFns.format(stats.date, 'd MMM')}
+
+ {readableQuantity(stats.combined.sum)} (+ + {readableQuantity(stats.combined.value)}) +
+
+
+ {stats.byExt.map((s) => { + const hl = stats.highlighted ? stats.highlighted.ext === s.ext : true + return ( + +
{s.ext || 'other'}
+
+
+ {readableQuantity(s.sum)} (+ + {readableQuantity(s.value)}) +
+ + ) + })} +
+ + ) +} + +interface TransitionProps { + children: JSX.Element + in: boolean +} + +function Transition({ children, ...props }: TransitionProps) { + const contentsRef = React.useRef(null) + // when `in` is false, we want to keep the last rendered contents + if (props.in) contentsRef.current = children + return contentsRef.current && {contentsRef.current} +} + +const useStyles = M.makeStyles((t) => ({ + root: { + display: 'grid', + gridRowGap: t.spacing(0.25), + gridTemplateAreas: ` + "heading period" + "chart chart" + `, + gridTemplateColumns: 'min-content 1fr', + gridTemplateRows: 'auto auto', + [t.breakpoints.down('sm')]: { + gridTemplateAreas: ` + "heading" + "chart" + "period" + `, + gridTemplateColumns: '1fr', + gridTemplateRows: 'auto auto auto', + }, + }, + heading: { + ...t.typography.h6, + gridArea: 'heading', + marginBottom: t.spacing(1), + whiteSpace: 'nowrap', + [t.breakpoints.down('sm')]: { + marginBottom: 0, + textAlign: 'center', + }, + }, + ext: { + display: 'inline-block', + maxWidth: 100, + overflow: 'hidden', + textOverflow: 'ellipsis', + verticalAlign: 'bottom', + }, + period: { + display: 'flex', + gridArea: 'period', + justifyContent: 'center', + alignItems: 'center', + [t.breakpoints.down('sm')]: { + paddingBottom: t.spacing(1), + paddingTop: t.spacing(2), + }, + [t.breakpoints.up('md')]: { + height: 37, + justifyContent: 'flex-end', + }, + }, + chart: { + gridArea: 'chart', + position: 'relative', + }, + left: {}, + right: {}, + dateStats: { + maxWidth: 180, + position: 'absolute', + top: 0, + width: 'calc(50% - 8px)', + zIndex: 1, + '&$left': { + left: 0, + }, + '&$right': { + right: 0, + }, + }, + unavail: { + ...t.typography.body2, + alignItems: 'center', + display: 'flex', + height: '100%', + justifyContent: 'center', + position: 'absolute', + top: 0, + width: '100%', + }, +})) + +interface DownloadsProps extends M.BoxProps { + bucket: string + colorPool: ColorPool + chartHeight: number +} + +export default function Downloads({ + bucket, + colorPool, + chartHeight, + ...props +}: DownloadsProps) { + const classes = useStyles() + const ref = React.useRef(null) + const { width } = useComponentSize(ref) + const [window, setWindow] = React.useState(ANALYTICS_WINDOW_OPTIONS[0].value) + + const [cursor, setCursor] = React.useState(null) + + const result = GQL.useQuery( + BUCKET_ACCESS_COUNTS_QUERY, + { bucket, window }, + { pause: !cfg.analyticsBucket }, + ) + + const processed = React.useMemo( + () => + Eff.pipe( + result, + ({ fetching, data, error }) => { + if (fetching) return Eff.Option.none() + if (error) log.error('Error fetching bucket access counts:', error) + return Eff.Option.fromNullable(data?.bucketAccessCounts) + }, + Eff.Option.map(processBucketAccessCounts), + ), + [result], + ) + + const processedWithCursor = React.useMemo( + () => + Eff.pipe( + processed, + Eff.Option.map((counts) => ({ + counts, + cursorStats: getCursorStats(counts, cursor), + })), + ), + [processed, cursor], + ) + + if (!cfg.analyticsBucket) { + return ( + +
Requires CloudTrail
+ + ) + } + + return ( + +
+ +
+
+ {Eff.Option.match(processedWithCursor, { + onSome: ({ counts, cursorStats: stats }) => { + if (!counts?.byExtCollapsed.length) return 'Downloads' + + const hl = stats?.highlighted + const ext = hl ? hl.ext || 'other' : 'total' + const total = hl ? hl.counts.total : counts.combined.total + return ( + <> + Downloads ({ext}):{' '} + {readableQuantity(total)} + + ) + }, + onNone: () => 'Downloads', + })} +
+
+ {Eff.Option.match(processedWithCursor, { + onSome: ({ counts, cursorStats: stats }) => { + if (!counts.byExtCollapsed.length) { + return ( + +
No Data
+ + ) + } + + return ( + <> + {/* @ts-expect-error */} + + e.counts.counts.map((i) => Math.log(i.sum + 1)), + )} + onCursor={setCursor} + height={chartHeight} + width={width} + areaFills={counts.byExtCollapsed.map((e) => + SVG.Paint.Color(colorPool.get(e.ext)), + )} + lineStroke={SVG.Paint.Color(M.colors.grey[500])} + extendL + extendR + px={10} + /> + + + + + + + + ) + }, + onNone: () => , + })} +
+ + ) +} diff --git a/catalog/app/containers/Bucket/Overview/Header.tsx b/catalog/app/containers/Bucket/Overview/Header.tsx new file mode 100644 index 00000000000..3c76edd1330 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/Header.tsx @@ -0,0 +1,431 @@ +import type AWSSDK from 'aws-sdk' +import cx from 'classnames' +import * as Eff from 'effect' +import * as React from 'react' +import { Link as RRLink } from 'react-router-dom' +import * as redux from 'react-redux' +import * as M from '@material-ui/core' + +import Skeleton from 'components/Skeleton' +import * as authSelectors from 'containers/Auth/selectors' +import * as APIConnector from 'utils/APIConnector' +import AsyncResult from 'utils/AsyncResult' +import { useData } from 'utils/Data' +import * as NamedRoutes from 'utils/NamedRoutes' +import { readableBytes, readableQuantity, formatQuantity } from 'utils/string' +import useConst from 'utils/useConstant' + +import * as requests from '../requests' + +import { ColorPool, makeColorPool } from './ColorPool' +import Downloads from './Downloads' + +import bg from './Overview-bg.jpg' + +// interface StatsData { +// exts: ExtData[] +// totalObjects: number +// totalBytes: number +// } + +interface ExtData { + ext: string + bytes: number + objects: number +} + +const RODA_LINK = 'https://registry.opendata.aws' +const RODA_BUCKET = 'quilt-open-data-bucket' +const MAX_EXTS = 7 +// must have length >= MAX_EXTS +const COLOR_MAP = [ + '#8ad3cb', + '#d7ce69', + '#bfbadb', + '#f4806c', + '#83b0d1', + '#b2de67', + '#bc81be', + '#f0b5d3', + '#7ba39f', + '#9894ad', + '#be7265', + '#94ad6b', +] + +const useObjectsByExtStyles = M.makeStyles((t) => ({ + root: { + display: 'grid', + gridAutoRows: 20, + gridColumnGap: t.spacing(1), + gridRowGap: t.spacing(0.25), + gridTemplateAreas: ` + ". heading heading" + `, + gridTemplateColumns: 'minmax(30px, max-content) 1fr minmax(30px, max-content)', + gridTemplateRows: 'auto', + [t.breakpoints.down('sm')]: { + gridTemplateAreas: ` + "heading heading heading" + `, + }, + }, + heading: { + ...t.typography.h6, + gridArea: 'heading', + marginBottom: t.spacing(1), + [t.breakpoints.down('sm')]: { + textAlign: 'center', + }, + }, + ext: { + color: t.palette.text.secondary, + gridColumn: 1, + fontSize: t.typography.overline.fontSize, + fontWeight: t.typography.fontWeightMedium, + letterSpacing: t.typography.subtitle2.letterSpacing, + lineHeight: t.typography.pxToRem(20), + textAlign: 'right', + }, + count: { + color: t.palette.text.secondary, + gridColumn: 3, + fontSize: t.typography.overline.fontSize, + fontWeight: t.typography.fontWeightMedium, + letterSpacing: t.typography.subtitle2.letterSpacing, + lineHeight: t.typography.pxToRem(20), + }, + bar: { + background: t.palette.action.hover, + gridColumn: 2, + }, + gauge: { + height: '100%', + position: 'relative', + }, + flip: {}, + size: { + color: t.palette.common.white, + fontSize: t.typography.overline.fontSize, + fontWeight: t.typography.fontWeightMedium, + letterSpacing: t.typography.subtitle2.letterSpacing, + lineHeight: t.typography.pxToRem(20), + position: 'absolute', + right: t.spacing(1), + '&$flip': { + color: t.palette.text.hint, + left: `calc(100% + ${t.spacing(1)}px)`, + right: 'auto', + }, + }, + skeleton: { + gridColumn: '1 / span 3', + }, + unavail: { + ...t.typography.body2, + alignItems: 'center', + display: 'flex', + gridColumn: '1 / span 3', + gridRow: `2 / span ${MAX_EXTS}`, + justifyContent: 'center', + }, +})) + +interface ObjectsByExtProps extends M.BoxProps { + data: $TSFixMe // AsyncResult + colorPool: ColorPool +} + +function ObjectsByExt({ data, colorPool, ...props }: ObjectsByExtProps) { + const classes = useObjectsByExtStyles() + return ( + +
Objects by File Extension
+ {AsyncResult.case( + { + Ok: (exts: ExtData[]) => { + const capped = exts.slice(0, MAX_EXTS) + const maxBytes = capped.reduce((max, e) => Math.max(max, e.bytes), 0) + const max = Math.log(maxBytes + 1) + const scale = (x: number) => Math.log(x + 1) / max + return capped.map(({ ext, bytes, objects }, i) => { + const color = colorPool.get(ext) + return ( + +
+ {ext || 'other'} +
+
+
+
+ {readableBytes(bytes)} +
+
+
+
+ {readableQuantity(objects)} +
+ + ) + }) + }, + _: (r: $TSFixMe) => ( + <> + {Eff.Array.makeBy(MAX_EXTS, (i) => ( + + ))} + {AsyncResult.Err.is(r) && ( +
Data unavailable
+ )} + + ), + }, + data, + )} + + ) +} + +const useStatDisplayStyles = M.makeStyles((t) => ({ + root: { + alignItems: 'baseline', + display: 'flex', + '& + &': { + marginLeft: t.spacing(1.5), + [t.breakpoints.up('sm')]: { + marginLeft: t.spacing(4), + }, + [t.breakpoints.up('md')]: { + marginLeft: t.spacing(6), + }, + }, + }, + value: { + fontSize: t.typography.h6.fontSize, + fontWeight: t.typography.fontWeightBold, + letterSpacing: 0, + lineHeight: '20px', + [t.breakpoints.up('sm')]: { + fontSize: t.typography.h4.fontSize, + lineHeight: '32px', + }, + }, + label: { + ...t.typography.body2, + color: t.palette.grey[300], + lineHeight: 1, + marginLeft: t.spacing(0.5), + [t.breakpoints.up('sm')]: { + marginLeft: t.spacing(1), + }, + }, + skeletonContainer: { + alignItems: 'center', + height: 20, + [t.breakpoints.up('sm')]: { + height: 32, + }, + }, + skeleton: { + borderRadius: t.shape.borderRadius, + height: t.typography.h6.fontSize, + width: 96, + [t.breakpoints.up('sm')]: { + height: t.typography.h4.fontSize, + width: 120, + }, + }, +})) + +interface StatDisplayProps { + value: $TSFixMe // AsyncResult + label?: string + format?: (v: any) => any + fallback?: (v: any) => any +} + +function StatDisplay({ value, label, format, fallback }: StatDisplayProps) { + const classes = useStatDisplayStyles() + return Eff.pipe( + value, + AsyncResult.case({ + Ok: Eff.flow(format || Eff.identity, AsyncResult.Ok), + Err: Eff.flow(fallback || Eff.identity, AsyncResult.Ok), + _: Eff.identity, + }), + AsyncResult.case({ + Ok: (v: $TSFixMe) => + v != null && ( + + {v} + {!!label && {label}} + + ), + _: () => ( +
+ +
+ ), + }), + ) as JSX.Element +} + +// use the same height as the bar chart: 20px per bar with 2px margin +const DOWNLOADS_CHART_H = 22 * MAX_EXTS - 2 + +const useStyles = M.makeStyles((t) => ({ + root: { + position: 'relative', + [t.breakpoints.down('xs')]: { + borderRadius: 0, + }, + [t.breakpoints.up('sm')]: { + marginTop: t.spacing(2), + }, + }, + top: { + background: `center / cover url(${bg}) ${t.palette.grey[700]}`, + borderTopLeftRadius: t.shape.borderRadius, + borderTopRightRadius: t.shape.borderRadius, + color: t.palette.common.white, + overflow: 'hidden', + paddingBottom: t.spacing(3), + paddingLeft: t.spacing(2), + paddingRight: t.spacing(2), + paddingTop: t.spacing(4), + position: 'relative', + [t.breakpoints.up('sm')]: { + padding: t.spacing(4), + }, + [t.breakpoints.down('xs')]: { + borderRadius: 0, + }, + }, + settings: { + color: t.palette.common.white, + position: 'absolute', + right: t.spacing(2), + top: t.spacing(2), + }, +})) + +interface HeaderProps { + s3: AWSSDK.S3 + bucket: string + overviewUrl: string | null | undefined + description: string | null | undefined +} + +export default function Header({ s3, overviewUrl, bucket, description }: HeaderProps) { + const classes = useStyles() + const req = APIConnector.use() + const isRODA = !!overviewUrl && overviewUrl.includes(`/${RODA_BUCKET}/`) + const colorPool = useConst(() => makeColorPool(COLOR_MAP)) + const statsData = useData(requests.bucketStats, { req, s3, bucket, overviewUrl }) + const pkgCountData = useData(requests.countPackageRevisions, { req, bucket }) + const { urls } = NamedRoutes.use() + const isAdmin = redux.useSelector(authSelectors.isAdmin) + return ( + + + {bucket} + {!!description && ( + + {description} + + )} + {isRODA && ( + + + From the{' '} + + Registry of Open Data on AWS + + + + )} + + '? B'} + /> + '?'} + /> + null} + /> + + {isAdmin && ( + + + settings + + + )} + + + + + + + + + + + + ) +} diff --git a/catalog/app/containers/Bucket/Overview-bg.jpg b/catalog/app/containers/Bucket/Overview/Overview-bg.jpg similarity index 100% rename from catalog/app/containers/Bucket/Overview-bg.jpg rename to catalog/app/containers/Bucket/Overview/Overview-bg.jpg diff --git a/catalog/app/containers/Bucket/Overview/Overview.tsx b/catalog/app/containers/Bucket/Overview/Overview.tsx new file mode 100644 index 00000000000..2eee868fd4f --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/Overview.tsx @@ -0,0 +1,163 @@ +import type AWSSDK from 'aws-sdk' +import * as React from 'react' +import { useParams } from 'react-router-dom' +import * as M from '@material-ui/core' + +import cfg from 'constants/config' +import type * as Model from 'model' +import * as APIConnector from 'utils/APIConnector' +import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' +import * as BucketPreferences from 'utils/BucketPreferences' +import Data from 'utils/Data' +import * as GQL from 'utils/GraphQL' +import * as LinkedData from 'utils/LinkedData' + +import * as Gallery from '../Gallery' +import * as Summarize from '../Summarize' +import * as requests from '../requests' + +import Header from './Header' +import BUCKET_CONFIG_QUERY from './gql/BucketConfig.generated' + +interface BucketReadmes { + forced?: Model.S3.S3ObjectLocation + discovered: Model.S3.S3ObjectLocation[] +} + +interface ReadmesProps { + s3: AWSSDK.S3 + bucket: string + overviewUrl: string | undefined | null +} + +function Readmes({ s3, overviewUrl, bucket }: ReadmesProps) { + return ( + // @ts-expect-error + + {AsyncResult.case({ + Ok: (rs: BucketReadmes) => + (rs.discovered.length > 0 || !!rs.forced) && ( + <> + {!!rs.forced && ( + + )} + {rs.discovered.map((h) => ( + + ))} + + ), + _: () => , + })} + + ) +} + +interface ImgsProps { + s3: AWSSDK.S3 + bucket: string + overviewUrl: string | undefined | null + inStack: boolean +} + +function Imgs({ s3, overviewUrl, inStack, bucket }: ImgsProps) { + const req = APIConnector.use() + return ( + // @ts-expect-error + + {AsyncResult.case({ + Ok: (images: Model.S3.S3ObjectLocation[]) => + images.length ? : null, + _: () => , + })} + + ) +} + +interface ThumbnailsWrapperProps extends ImgsProps { + preferences?: + | false + | { + overview: boolean + summarize: boolean + } +} + +function ThumbnailsWrapper({ + s3, + overviewUrl, + inStack, + bucket, + preferences: galleryPrefs, +}: ThumbnailsWrapperProps) { + if (cfg.noOverviewImages || !galleryPrefs) return null + if (!galleryPrefs.overview) return null + return ( + // @ts-expect-error + + {AsyncResult.case({ + Ok: (h?: Model.S3.S3ObjectLocation) => + (!h || galleryPrefs.summarize) && ( + + ), + Err: () => , + Pending: () => , + _: () => null, + })} + + ) +} + +export default function Overview() { + const { bucket } = useParams<{ bucket: string }>() + + const s3 = AWS.S3.use() + const { bucketConfig } = GQL.useQueryS(BUCKET_CONFIG_QUERY, { bucket }) + const inStack = !!bucketConfig + const overviewUrl = bucketConfig?.overviewUrl + const description = bucketConfig?.description + const prefs = BucketPreferences.use() + return ( + + {inStack && ( + + + + )} + {bucketConfig ? ( +
+ ) : ( + + {bucket} + + )} + + {BucketPreferences.Result.match( + { + Ok: ({ ui: { blocks } }) => ( + + ), + Pending: () => , + Init: () => null, + }, + prefs, + )} + + + ) +} diff --git a/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.generated.ts b/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.generated.ts new file mode 100644 index 00000000000..f7a763654e1 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.generated.ts @@ -0,0 +1,207 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core' +import * as Types from '../../../../model/graphql/types.generated' + +export type AccessCountsSelectionFragment = { + readonly __typename: 'AccessCounts' +} & Pick & { + readonly counts: ReadonlyArray< + { readonly __typename: 'AccessCountForDate' } & Pick< + Types.AccessCountForDate, + 'date' | 'value' + > + > + } + +export type containers_Bucket_Overview_gql_BucketAccessCountsQueryVariables = + Types.Exact<{ + bucket: Types.Scalars['String'] + window: Types.Scalars['Int'] + }> + +export type containers_Bucket_Overview_gql_BucketAccessCountsQuery = { + readonly __typename: 'Query' +} & { + readonly bucketAccessCounts: Types.Maybe< + { readonly __typename: 'BucketAccessCounts' } & { + readonly byExt: ReadonlyArray< + { readonly __typename: 'AccessCountsGroup' } & Pick< + Types.AccessCountsGroup, + 'ext' + > & { + readonly counts: { + readonly __typename: 'AccessCounts' + } & AccessCountsSelectionFragment + } + > + readonly byExtCollapsed: ReadonlyArray< + { readonly __typename: 'AccessCountsGroup' } & Pick< + Types.AccessCountsGroup, + 'ext' + > & { + readonly counts: { + readonly __typename: 'AccessCounts' + } & AccessCountsSelectionFragment + } + > + readonly combined: { + readonly __typename: 'AccessCounts' + } & AccessCountsSelectionFragment + } + > +} + +export const AccessCountsSelectionFragmentDoc = { + kind: 'Document', + definitions: [ + { + kind: 'FragmentDefinition', + name: { kind: 'Name', value: 'AccessCountsSelection' }, + typeCondition: { kind: 'NamedType', name: { kind: 'Name', value: 'AccessCounts' } }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'total' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'counts' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'date' } }, + { kind: 'Field', name: { kind: 'Name', value: 'value' } }, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode +export const containers_Bucket_Overview_gql_BucketAccessCountsDocument = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'containers_Bucket_Overview_gql_BucketAccessCounts' }, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: { kind: 'Variable', name: { kind: 'Name', value: 'bucket' } }, + type: { + kind: 'NonNullType', + type: { kind: 'NamedType', name: { kind: 'Name', value: 'String' } }, + }, + }, + { + kind: 'VariableDefinition', + variable: { kind: 'Variable', name: { kind: 'Name', value: 'window' } }, + type: { + kind: 'NonNullType', + type: { kind: 'NamedType', name: { kind: 'Name', value: 'Int' } }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'bucketAccessCounts' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'bucket' }, + value: { kind: 'Variable', name: { kind: 'Name', value: 'bucket' } }, + }, + { + kind: 'Argument', + name: { kind: 'Name', value: 'window' }, + value: { kind: 'Variable', name: { kind: 'Name', value: 'window' } }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'byExt' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'ext' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'counts' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'FragmentSpread', + name: { kind: 'Name', value: 'AccessCountsSelection' }, + }, + ], + }, + }, + ], + }, + }, + { + kind: 'Field', + alias: { kind: 'Name', value: 'byExtCollapsed' }, + name: { kind: 'Name', value: 'byExt' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'groups' }, + value: { kind: 'IntValue', value: '10' }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'ext' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'counts' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'FragmentSpread', + name: { kind: 'Name', value: 'AccessCountsSelection' }, + }, + ], + }, + }, + ], + }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'combined' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'FragmentSpread', + name: { kind: 'Name', value: 'AccessCountsSelection' }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ...AccessCountsSelectionFragmentDoc.definitions, + ], +} as unknown as DocumentNode< + containers_Bucket_Overview_gql_BucketAccessCountsQuery, + containers_Bucket_Overview_gql_BucketAccessCountsQueryVariables +> + +export { containers_Bucket_Overview_gql_BucketAccessCountsDocument as default } diff --git a/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.graphql b/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.graphql new file mode 100644 index 00000000000..c54990cda53 --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/gql/BucketAccessCounts.graphql @@ -0,0 +1,27 @@ +fragment AccessCountsSelection on AccessCounts { + total + counts { + date + value + } +} + +query ($bucket: String!, $window: Int!) { + bucketAccessCounts(bucket: $bucket, window: $window) { + byExt { + ext + counts { + ...AccessCountsSelection + } + } + byExtCollapsed: byExt(groups: 10) { + ext + counts { + ...AccessCountsSelection + } + } + combined { + ...AccessCountsSelection + } + } +} diff --git a/catalog/app/containers/Bucket/OverviewBucketConfig.generated.ts b/catalog/app/containers/Bucket/Overview/gql/BucketConfig.generated.ts similarity index 75% rename from catalog/app/containers/Bucket/OverviewBucketConfig.generated.ts rename to catalog/app/containers/Bucket/Overview/gql/BucketConfig.generated.ts index 89a16de328f..293100b338b 100644 --- a/catalog/app/containers/Bucket/OverviewBucketConfig.generated.ts +++ b/catalog/app/containers/Bucket/Overview/gql/BucketConfig.generated.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core' -import * as Types from '../../model/graphql/types.generated' +import * as Types from '../../../../model/graphql/types.generated' -export type containers_Bucket_OverviewBucketConfigQueryVariables = Types.Exact<{ +export type containers_Bucket_Overview_gql_BucketConfigQueryVariables = Types.Exact<{ bucket: Types.Scalars['String'] }> -export type containers_Bucket_OverviewBucketConfigQuery = { +export type containers_Bucket_Overview_gql_BucketConfigQuery = { readonly __typename: 'Query' } & { readonly bucketConfig: Types.Maybe< @@ -17,13 +17,13 @@ export type containers_Bucket_OverviewBucketConfigQuery = { > } -export const containers_Bucket_OverviewBucketConfigDocument = { +export const containers_Bucket_Overview_gql_BucketConfigDocument = { kind: 'Document', definitions: [ { kind: 'OperationDefinition', operation: 'query', - name: { kind: 'Name', value: 'containers_Bucket_OverviewBucketConfig' }, + name: { kind: 'Name', value: 'containers_Bucket_Overview_gql_BucketConfig' }, variableDefinitions: [ { kind: 'VariableDefinition', @@ -61,8 +61,8 @@ export const containers_Bucket_OverviewBucketConfigDocument = { }, ], } as unknown as DocumentNode< - containers_Bucket_OverviewBucketConfigQuery, - containers_Bucket_OverviewBucketConfigQueryVariables + containers_Bucket_Overview_gql_BucketConfigQuery, + containers_Bucket_Overview_gql_BucketConfigQueryVariables > -export { containers_Bucket_OverviewBucketConfigDocument as default } +export { containers_Bucket_Overview_gql_BucketConfigDocument as default } diff --git a/catalog/app/containers/Bucket/OverviewBucketConfig.graphql b/catalog/app/containers/Bucket/Overview/gql/BucketConfig.graphql similarity index 100% rename from catalog/app/containers/Bucket/OverviewBucketConfig.graphql rename to catalog/app/containers/Bucket/Overview/gql/BucketConfig.graphql diff --git a/catalog/app/containers/Bucket/Overview/index.tsx b/catalog/app/containers/Bucket/Overview/index.tsx new file mode 100644 index 00000000000..1de667af70e --- /dev/null +++ b/catalog/app/containers/Bucket/Overview/index.tsx @@ -0,0 +1 @@ +export { default } from './Overview' diff --git a/catalog/app/containers/Bucket/Summarize.tsx b/catalog/app/containers/Bucket/Summarize.tsx index e644215263c..b88512727dd 100644 --- a/catalog/app/containers/Bucket/Summarize.tsx +++ b/catalog/app/containers/Bucket/Summarize.tsx @@ -258,7 +258,7 @@ interface FilePreviewProps { expanded?: boolean file?: SummarizeFile handle: LogicalKeyResolver.S3SummarizeHandle - headingOverride: React.ReactNode + headingOverride?: React.ReactNode packageHandle?: PackageHandle } @@ -270,7 +270,7 @@ export function FilePreview({ packageHandle, }: FilePreviewProps) { const description = file?.description ? : null - const heading = headingOverride != null ? headingOverride : + const heading = headingOverride ?? const key = handle.logicalKey || handle.key const props = React.useMemo(() => Preview.getRenderProps(key, file), [key, file]) @@ -566,7 +566,7 @@ interface SummaryRootProps { s3: S3 bucket: string inStack: boolean - overviewUrl: string + overviewUrl?: string | null } export function SummaryRoot({ s3, bucket, inStack, overviewUrl }: SummaryRootProps) { diff --git a/catalog/app/containers/Bucket/requests/requestsUntyped.js b/catalog/app/containers/Bucket/requests/requestsUntyped.js index 2ba9722da61..5efb639049f 100644 --- a/catalog/app/containers/Bucket/requests/requestsUntyped.js +++ b/catalog/app/containers/Bucket/requests/requestsUntyped.js @@ -1,7 +1,6 @@ import { join as pathJoin } from 'path' -import * as dateFns from 'date-fns' -import * as FP from 'fp-ts' +import * as Eff from 'effect' import sampleSize from 'lodash/fp/sampleSize' import * as R from 'ramda' @@ -9,7 +8,6 @@ import quiltSummarizeSchema from 'schemas/quilt_summarize.json' import { SUPPORTED_EXTENSIONS as IMG_EXTS } from 'components/Thumbnail' import * as quiltConfigs from 'constants/quiltConfigs' -import cfg from 'constants/config' import * as Resource from 'utils/Resource' import { makeSchemaValidator } from 'utils/json-schema' import mkSearch from 'utils/mkSearch' @@ -24,106 +22,6 @@ import { decodeS3Key } from './utils' const promiseProps = (obj) => Promise.all(Object.values(obj)).then(R.zipObj(Object.keys(obj))) -const MAX_BANDS = 10 - -export const bucketAccessCounts = async ({ s3, bucket, today, window }) => { - if (!cfg.analyticsBucket) - throw new Error('bucketAccessCounts: "analyticsBucket" required') - - const dates = R.unfold( - (daysLeft) => daysLeft >= 0 && [dateFns.subDays(today, daysLeft), daysLeft - 1], - window, - ) - - try { - const result = await s3Select({ - s3, - Bucket: cfg.analyticsBucket, - Key: `${ACCESS_COUNTS_PREFIX}/Exts.csv`, - Expression: ` - SELECT ext, counts FROM s3object - WHERE eventname = 'GetObject' - AND bucket = '${sqlEscape(bucket)}' - `, - InputSerialization: { - CSV: { - FileHeaderInfo: 'Use', - AllowQuotedRecordDelimiter: true, - }, - }, - }) - return FP.function.pipe( - result, - R.map((r) => { - const recordedCounts = JSON.parse(r.counts) - const { counts, total } = dates.reduce( - (acc, date) => { - const value = recordedCounts[dateFns.format(date, 'yyyy-MM-dd')] || 0 - const sum = acc.total + value - return { - total: sum, - counts: acc.counts.concat({ date, value, sum }), - } - }, - { total: 0, counts: [] }, - ) - return { ext: r.ext && `.${r.ext}`, total, counts } - }), - R.filter((i) => i.total), - R.sort(R.descend(R.prop('total'))), - R.applySpec({ - byExt: R.identity, - byExtCollapsed: (bands) => { - if (bands.length <= MAX_BANDS) return bands - const [other, rest] = R.partition((b) => b.ext === '', bands) - const [toKeep, toMerge] = R.splitAt(MAX_BANDS - 1, rest) - const merged = [...other, ...toMerge].reduce((acc, band) => ({ - ext: '', - total: acc.total + band.total, - counts: R.zipWith( - (a, b) => ({ - date: a.date, - value: a.value + b.value, - sum: a.sum + b.sum, - }), - acc.counts, - band.counts, - ), - })) - return R.sort(R.descend(R.prop('total')), toKeep.concat(merged)) - }, - combined: { - total: R.reduce((sum, { total }) => sum + total, 0), - counts: R.pipe( - R.pluck('counts'), - R.transpose, - R.map( - R.reduce( - (acc, { date, value, sum }) => ({ - date, - value: acc.value + value, - sum: acc.sum + sum, - }), - { value: 0, sum: 0 }, - ), - ), - ), - }, - }), - ) - } catch (e) { - // eslint-disable-next-line no-console - console.log('Unable to fetch bucket access counts:') - // eslint-disable-next-line no-console - console.error(e) - return { - byExt: [], - byExtCollapsed: [], - combined: { total: 0, counts: [] }, - } - } -} - const parseDate = (d) => d && new Date(d) const getOverviewBucket = (url) => s3paths.parseS3Url(url).bucket @@ -373,7 +271,7 @@ export const bucketSummary = async ({ s3, req, bucket, overviewUrl, inStack }) = Key: getOverviewKey(overviewUrl, 'summary.json'), }) .promise() - return FP.function.pipe( + return Eff.pipe( JSON.parse(r.Body.toString('utf-8')), R.pathOr([], ['aggregations', 'other', 'keys', 'buckets']), R.map((b) => ({ @@ -403,7 +301,7 @@ export const bucketSummary = async ({ s3, req, bucket, overviewUrl, inStack }) = try { const qs = mkSearch({ action: 'sample', index: bucket }) const result = await req(`/search${qs}`) - return FP.function.pipe( + return Eff.pipe( result, R.pathOr([], ['aggregations', 'objects', 'buckets']), R.map((h) => { @@ -425,7 +323,7 @@ export const bucketSummary = async ({ s3, req, bucket, overviewUrl, inStack }) = const result = await s3 .listObjectsV2({ Bucket: bucket, EncodingType: 'url' }) .promise() - return FP.function.pipe( + return Eff.pipe( result, R.path(['Contents']), R.map(R.evolve({ Key: decodeS3Key })), @@ -477,7 +375,7 @@ export const bucketImgs = async ({ req, s3, bucket, overviewUrl, inStack }) => { Key: getOverviewKey(overviewUrl, 'summary.json'), }) .promise() - return FP.function.pipe( + return Eff.pipe( JSON.parse(r.Body.toString('utf-8')), R.pathOr([], ['aggregations', 'images', 'keys', 'buckets']), R.map((b) => ({ @@ -498,7 +396,7 @@ export const bucketImgs = async ({ req, s3, bucket, overviewUrl, inStack }) => { try { const qs = mkSearch({ action: 'images', index: bucket }) const result = await req(`/search${qs}`) - return FP.function.pipe( + return Eff.pipe( result, R.pathOr([], ['aggregations', 'objects', 'buckets']), R.map((h) => { @@ -519,7 +417,7 @@ export const bucketImgs = async ({ req, s3, bucket, overviewUrl, inStack }) => { const result = await s3 .listObjectsV2({ Bucket: bucket, EncodingType: 'url' }) .promise() - return FP.function.pipe( + return Eff.pipe( result, R.path(['Contents']), R.map(R.evolve({ Key: decodeS3Key })), @@ -656,8 +554,6 @@ export const summarize = async ({ s3, handle: inputHandle, resolveLogicalKey }) } } -const MANIFESTS_PREFIX = '.quilt/packages/' - const withCalculatedRevisions = (s) => ({ scripted_metric: { init_script: ` @@ -712,113 +608,33 @@ export const countPackageRevisions = ({ req, bucket, name }) => .then(R.path(['aggregations', 'revisions', 'value'])) .catch(errors.catchErrors()) -// TODO: Preview endpoint only allows up to 512 lines right now. Increase it to 1000. -const MAX_PACKAGE_ENTRIES = 500 - -// TODO: remove -export const getRevisionData = async ({ - endpoint, - sign, - bucket, - hash, - maxKeys = MAX_PACKAGE_ENTRIES, -}) => { - const url = sign({ bucket, key: `${MANIFESTS_PREFIX}${hash}` }) - const maxLines = maxKeys + 2 // 1 for the meta and 1 for checking overflow - const r = await fetch( - `${endpoint}/preview?url=${encodeURIComponent(url)}&input=txt&line_count=${maxLines}`, - ) - const [header, ...entries] = await r - .json() - .then((json) => json.info.data.head.map((l) => JSON.parse(l))) - const files = Math.min(maxKeys, entries.length) - const bytes = entries.slice(0, maxKeys).reduce((sum, i) => sum + i.size, 0) - const truncated = entries.length > maxKeys - return { - stats: { files, bytes, truncated }, - message: header.message, - header, - } -} - -const s3Select = ({ - s3, - ExpressionType = 'SQL', - InputSerialization = { JSON: { Type: 'LINES' } }, - ...rest -}) => - s3 - .selectObjectContent({ - ExpressionType, - InputSerialization, - OutputSerialization: { JSON: {} }, - ...rest, - }) - .promise() - .then( - R.pipe( - R.prop('Payload'), - R.reduce((acc, evt) => { - if (!evt.Records) return acc - const s = evt.Records.Payload.toString() - return acc + s - }, ''), - R.trim, - R.ifElse(R.isEmpty, R.always([]), R.pipe(R.split('\n'), R.map(JSON.parse))), - ), - ) - -const sqlEscape = (arg) => arg.replace(/'/g, "''") +// const MANIFESTS_PREFIX = '.quilt/packages/' -const ACCESS_COUNTS_PREFIX = 'AccessCounts' - -const queryAccessCounts = async ({ s3, type, query, today, window = 365 }) => { - try { - const records = await s3Select({ - s3, - Bucket: cfg.analyticsBucket, - Key: `${ACCESS_COUNTS_PREFIX}/${type}.csv`, - Expression: query, - InputSerialization: { - CSV: { - FileHeaderInfo: 'Use', - AllowQuotedRecordDelimiter: true, - }, - }, - }) - - const recordedCounts = records.length ? JSON.parse(records[0].counts) : {} - - const counts = R.times((i) => { - const date = dateFns.subDays(today, window - i - 1) - return { - date, - value: recordedCounts[dateFns.format(date, 'yyyy-MM-dd')] || 0, - } - }, window) - - const total = Object.values(recordedCounts).reduce(R.add, 0) - - return { counts, total } - } catch (e) { - // eslint-disable-next-line no-console - console.log('queryAccessCounts: error caught') - // eslint-disable-next-line no-console - console.error(e) - throw e - } -} - -export const objectAccessCounts = ({ s3, bucket, path, today }) => - queryAccessCounts({ - s3, - type: 'Objects', - query: ` - SELECT counts FROM s3object - WHERE eventname = 'GetObject' - AND bucket = '${sqlEscape(bucket)}' - AND "key" = '${sqlEscape(path)}' - `, - today, - window: 365, - }) +// TODO: Preview endpoint only allows up to 512 lines right now. Increase it to 1000. +// const MAX_PACKAGE_ENTRIES = 500 + +// TODO: remove: used in a comented-out code in PackageList +// export const getRevisionData = async ({ +// endpoint, +// sign, +// bucket, +// hash, +// maxKeys = MAX_PACKAGE_ENTRIES, +// }) => { +// const url = sign({ bucket, key: `${MANIFESTS_PREFIX}${hash}` }) +// const maxLines = maxKeys + 2 // 1 for the meta and 1 for checking overflow +// const r = await fetch( +// `${endpoint}/preview?url=${encodeURIComponent(url)}&input=txt&line_count=${maxLines}`, +// ) +// const [header, ...entries] = await r +// .json() +// .then((json) => json.info.data.head.map((l) => JSON.parse(l))) +// const files = Math.min(maxKeys, entries.length) +// const bytes = entries.slice(0, maxKeys).reduce((sum, i) => sum + i.size, 0) +// const truncated = entries.length > maxKeys +// return { +// stats: { files, bytes, truncated }, +// message: header.message, +// header, +// } +// } diff --git a/catalog/app/embed/File.js b/catalog/app/embed/File.js index bc47739202b..247a19ff163 100644 --- a/catalog/app/embed/File.js +++ b/catalog/app/embed/File.js @@ -1,7 +1,5 @@ import { basename } from 'path' -import * as dateFns from 'date-fns' -import * as R from 'ramda' import * as React from 'react' import { Link, useLocation, useParams } from 'react-router-dom' import * as M from '@material-ui/core' @@ -9,22 +7,21 @@ import * as M from '@material-ui/core' import * as BreadCrumbs from 'components/BreadCrumbs' import Message from 'components/Message' import * as Preview from 'components/Preview' -import Sparkline from 'components/Sparkline' import cfg from 'constants/config' import * as Notifications from 'containers/Notifications' import * as AWS from 'utils/AWS' import AsyncResult from 'utils/AsyncResult' import { useData } from 'utils/Data' import * as NamedRoutes from 'utils/NamedRoutes' -import * as SVG from 'utils/SVG' import { linkStyle } from 'utils/StyledLink' import copyToClipboard from 'utils/clipboard' import * as Format from 'utils/format' import parseSearch from 'utils/parseSearch' import * as s3paths from 'utils/s3paths' -import { readableBytes, readableQuantity } from 'utils/string' +import { readableBytes } from 'utils/string' import FileCodeSamples from 'containers/Bucket/CodeSamples/File' +import Analytics from 'containers/Bucket/File/Analytics' import FileProperties from 'containers/Bucket/FileProperties' import * as FileView from 'containers/Bucket/FileView' import Section from 'containers/Bucket/Section' @@ -229,74 +226,6 @@ function VersionInfo({ bucket, path, version }) { ) } -function Analytics({ bucket, path }) { - const [cursor, setCursor] = React.useState(null) - const s3 = AWS.S3.use() - const today = React.useMemo(() => new Date(), []) - const formatDate = (date) => - dateFns.format( - date, - today.getFullYear() === date.getFullYear() ? 'd MMM' : 'd MMM yyyy', - ) - const data = useData(requests.objectAccessCounts, { - s3, - bucket, - path, - today, - }) - - const defaultExpanded = data.case({ - Ok: ({ total }) => !!total, - _: () => false, - }) - - return ( -
- {data.case({ - Ok: ({ counts, total }) => - total ? ( - - - Downloads - - {readableQuantity(cursor === null ? total : counts[cursor].value)} - - - {cursor === null - ? `${counts.length} days` - : formatDate(counts[cursor].date)} - - - - - - - , - )} - /> - - - ) : ( - No analytics available - ), - Err: () => No analytics available, - _: () => , - })} -
- ) -} - function CenteredProgress() { return ( diff --git a/catalog/app/model/graphql/schema.generated.ts b/catalog/app/model/graphql/schema.generated.ts index ba8ed87e3fd..be04791e97a 100644 --- a/catalog/app/model/graphql/schema.generated.ts +++ b/catalog/app/model/graphql/schema.generated.ts @@ -82,6 +82,41 @@ export default { ], interfaces: [], }, + { + kind: 'OBJECT', + name: 'AccessCountsGroup', + fields: [ + { + name: 'ext', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'String', + ofType: null, + }, + }, + args: [], + }, + { + name: 'counts', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'OBJECT', + name: 'AccessCounts', + ofType: null, + }, + }, + args: [], + }, + ], + interfaces: [], + }, + { + kind: 'SCALAR', + name: 'String', + }, { kind: 'OBJECT', name: 'AdminMutations', @@ -208,10 +243,6 @@ export default { ], interfaces: [], }, - { - kind: 'SCALAR', - name: 'String', - }, { kind: 'OBJECT', name: 'AdminQueries', @@ -365,6 +396,52 @@ export default { }, ], }, + { + kind: 'OBJECT', + name: 'BucketAccessCounts', + fields: [ + { + name: 'byExt', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'LIST', + ofType: { + kind: 'NON_NULL', + ofType: { + kind: 'OBJECT', + name: 'AccessCountsGroup', + ofType: null, + }, + }, + }, + }, + args: [ + { + name: 'groups', + type: { + kind: 'SCALAR', + name: 'Int', + ofType: null, + }, + }, + ], + }, + { + name: 'combined', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'OBJECT', + name: 'AccessCounts', + ofType: null, + }, + }, + args: [], + }, + ], + interfaces: [], + }, { kind: 'UNION', name: 'BucketAddResult', @@ -4188,6 +4265,81 @@ export default { }, args: [], }, + { + name: 'bucketAccessCounts', + type: { + kind: 'OBJECT', + name: 'BucketAccessCounts', + ofType: null, + }, + args: [ + { + name: 'bucket', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'String', + ofType: null, + }, + }, + }, + { + name: 'window', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'Int', + ofType: null, + }, + }, + }, + ], + }, + { + name: 'objectAccessCounts', + type: { + kind: 'OBJECT', + name: 'AccessCounts', + ofType: null, + }, + args: [ + { + name: 'bucket', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'String', + ofType: null, + }, + }, + }, + { + name: 'key', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'String', + ofType: null, + }, + }, + }, + { + name: 'window', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'Int', + ofType: null, + }, + }, + }, + ], + }, { name: 'admin', type: { diff --git a/catalog/app/model/graphql/types.generated.ts b/catalog/app/model/graphql/types.generated.ts index 8ad7b159639..fb5d1b2a862 100644 --- a/catalog/app/model/graphql/types.generated.ts +++ b/catalog/app/model/graphql/types.generated.ts @@ -36,6 +36,12 @@ export interface AccessCounts { readonly counts: ReadonlyArray } +export interface AccessCountsGroup { + readonly __typename: 'AccessCountsGroup' + readonly ext: Scalars['String'] + readonly counts: AccessCounts +} + export interface AdminMutations { readonly __typename: 'AdminMutations' readonly user: UserAdminMutations @@ -89,6 +95,16 @@ export type BrowsingSessionDisposeResult = Ok | OperationError export type BrowsingSessionRefreshResult = BrowsingSession | InvalidInput | OperationError +export interface BucketAccessCounts { + readonly __typename: 'BucketAccessCounts' + readonly byExt: ReadonlyArray + readonly combined: AccessCounts +} + +export interface BucketAccessCountsbyExtArgs { + groups: Maybe +} + export interface BucketAddInput { readonly name: Scalars['String'] readonly title: Scalars['String'] @@ -864,6 +880,8 @@ export interface Query { readonly searchMoreObjects: ObjectsSearchMoreResult readonly searchMorePackages: PackagesSearchMoreResult readonly subscription: SubscriptionState + readonly bucketAccessCounts: Maybe + readonly objectAccessCounts: Maybe readonly admin: AdminQueries readonly policies: ReadonlyArray readonly policy: Maybe @@ -910,6 +928,17 @@ export interface QuerysearchMorePackagesArgs { size?: Maybe } +export interface QuerybucketAccessCountsArgs { + bucket: Scalars['String'] + window: Scalars['Int'] +} + +export interface QueryobjectAccessCountsArgs { + bucket: Scalars['String'] + key: Scalars['String'] + window: Scalars['Int'] +} + export interface QuerypolicyArgs { id: Scalars['ID'] } diff --git a/catalog/app/utils/AWS/S3.js b/catalog/app/utils/AWS/S3.js index a51c29b5566..6b052a52927 100644 --- a/catalog/app/utils/AWS/S3.js +++ b/catalog/app/utils/AWS/S3.js @@ -43,44 +43,28 @@ function useSmartS3() { return useConstant(() => { class SmartS3 extends S3 { - getReqType(req) { + shouldSign(req) { const bucket = req.params.Bucket if (cfg.mode === 'LOCAL') { - return 'signed' + return true } - if (isAuthenticated()) { - if ( - // sign if operation is not bucket-specific - // (not sure if there are any such operations that can be used from the browser) - !bucket || - cfg.analyticsBucket === bucket || + if ( + isAuthenticated() && + // sign if operation is not bucket-specific + // (not sure if there are any such operations that can be used from the browser) + (!bucket || cfg.serviceBucket === bucket || statusReportsBucket === bucket || - (cfg.mode !== 'OPEN' && isInStack(bucket)) - ) { - return 'signed' - } - } else if (req.operation === 'selectObjectContent') { - return 'select' + (cfg.mode !== 'OPEN' && isInStack(bucket))) + ) { + return true } - return 'unsigned' - } - - populateURI(req) { - if (req.service.getReqType(req) === 'select') { - return - } - super.populateURI(req) + return false } customRequestHandler(req) { - const b = req.params.Bucket - const type = this.getReqType(req) - - if (b) { - const endpoint = new AWS.Endpoint( - type === 'select' ? `${cfg.apiGatewayEndpoint}/s3select/` : cfg.s3Proxy, - ) + if (req.params.Bucket) { + const endpoint = new AWS.Endpoint(cfg.s3Proxy) req.on('sign', () => { if (req.httpRequest[PRESIGN]) return @@ -96,10 +80,7 @@ function useSmartS3() { const basePath = endpoint.path.replace(/\/$/, '') req.httpRequest.endpoint = endpoint - req.httpRequest.path = - type === 'select' - ? `${basePath}${origPath}` - : `${basePath}/${origEndpoint.host}${origPath}` + req.httpRequest.path = `${basePath}/${origEndpoint.host}${origPath}` }) req.on( 'retry', @@ -138,9 +119,8 @@ function useSmartS3() { if (forceProxy) { req.httpRequest[FORCE_PROXY] = true } - const type = this.getReqType(req) - if (type !== 'signed') { + if (!this.shouldSign(req)) { req.toUnauthenticated() } diff --git a/catalog/app/utils/AWS/Signer.js b/catalog/app/utils/AWS/Signer.js index 0c0c24b1ac8..404fe0f4d73 100644 --- a/catalog/app/utils/AWS/Signer.js +++ b/catalog/app/utils/AWS/Signer.js @@ -25,7 +25,7 @@ export function useS3Signer({ urlExpiration: exp, forceProxy = false } = {}) { const statusReportsBucket = useStatusReportsBucket() const s3 = S3.use() const inStackOrSpecial = React.useCallback( - (b) => isInStack(b) || cfg.analyticsBucket === b || statusReportsBucket === b, + (b) => isInStack(b) || statusReportsBucket === b, [isInStack, statusReportsBucket], ) return React.useCallback( diff --git a/catalog/app/utils/GraphQL/Provider.tsx b/catalog/app/utils/GraphQL/Provider.tsx index 592b71e58e6..05c34cd7238 100644 --- a/catalog/app/utils/GraphQL/Provider.tsx +++ b/catalog/app/utils/GraphQL/Provider.tsx @@ -90,6 +90,8 @@ export default function GraphQLProvider({ children }: React.PropsWithChildren<{} keys: { AccessCountForDate: () => null, AccessCounts: () => null, + AccessCountsGroup: () => null, + BucketAccessCounts: () => null, BucketConfig: (b) => b.name as string, Canary: (c) => c.name as string, Collaborator: (c) => c.username as string, diff --git a/shared/graphql/schema.graphql b/shared/graphql/schema.graphql index 0bb997e7809..ea342cd5806 100644 --- a/shared/graphql/schema.graphql +++ b/shared/graphql/schema.graphql @@ -222,6 +222,16 @@ type AccessCounts { counts: [AccessCountForDate!]! } +type AccessCountsGroup { + ext: String! + counts: AccessCounts! +} + +type BucketAccessCounts { + byExt(groups: Int): [AccessCountsGroup!]! + combined: AccessCounts! +} + type PackageDir { path: String! metadata: JsonRecord @@ -556,6 +566,9 @@ type Query { searchMorePackages(after: String!, size: Int = 30): PackagesSearchMoreResult! subscription: SubscriptionState! + bucketAccessCounts(bucket: String!, window: Int!): BucketAccessCounts + objectAccessCounts(bucket: String!, key: String!, window: Int!): AccessCounts + admin: AdminQueries! @admin policies: [Policy!]! @admin From 2eb3cfc41f6050a158d3e1cd221b1bd304cd80c5 Mon Sep 17 00:00:00 2001 From: Sergey Fedoseev Date: Tue, 19 Nov 2024 14:39:14 +0100 Subject: [PATCH 4/4] Fix some doc URLs in catalog (#4205) Signed-off-by: dependabot[bot] Co-authored-by: Maksim Chervonnyi Co-authored-by: Alexei Mochalov Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: QuiltSimon <116831980+QuiltSimon@users.noreply.github.com> Co-authored-by: Dr. Ernie Prabhakar <19791+drernie@users.noreply.github.com> Co-authored-by: Dr. Ernie Prabhakar --- catalog/CHANGELOG.md | 1 + .../QuiltConfigEditor/BucketPreferences.tsx | 5 ++++- .../FileEditor/QuiltConfigEditor/Workflows.tsx | 2 +- catalog/app/containers/Admin/Status/Status.tsx | 5 ++++- .../containers/Admin/UsersAndRoles/SsoConfig.tsx | 5 ++++- catalog/app/containers/Bucket/CodeSamples/Dir.tsx | 4 ++-- catalog/app/containers/Bucket/CodeSamples/File.tsx | 2 +- .../app/containers/Bucket/CodeSamples/Package.tsx | 14 +++++++------- .../Bucket/PackageDialog/DialogError.tsx | 2 +- .../Bucket/PackageDialog/SelectWorkflow.tsx | 2 +- .../Bucket/Queries/Athena/Workgroups.tsx | 6 ++++-- catalog/app/containers/Bucket/Successors.tsx | 4 ++-- catalog/app/containers/Bucket/Summarize.tsx | 4 +++- catalog/app/containers/Bucket/errors.tsx | 4 ++-- .../containers/NavBar/Suggestions/Suggestions.tsx | 5 ++++- 15 files changed, 41 insertions(+), 24 deletions(-) diff --git a/catalog/CHANGELOG.md b/catalog/CHANGELOG.md index 93888fa202c..ab6c3f62946 100644 --- a/catalog/CHANGELOG.md +++ b/catalog/CHANGELOG.md @@ -17,6 +17,7 @@ where verb is one of ## Changes +- [Fixed] Fix some doc URLs in catalog ([#4205](https://github.com/quiltdata/quilt/pull/4205)) - [Changed] S3 Select -> GQL API calls for getting access counts ([#4218](https://github.com/quiltdata/quilt/pull/4218)) - [Changed] Athena: improve loading state and errors visuals; fix minor bugs; alphabetize and persist selection in workgroups, catalog names and databases ([#4208](https://github.com/quiltdata/quilt/pull/4208)) - [Changed] Show stack release version in footer ([#4200](https://github.com/quiltdata/quilt/pull/4200)) diff --git a/catalog/app/components/FileEditor/QuiltConfigEditor/BucketPreferences.tsx b/catalog/app/components/FileEditor/QuiltConfigEditor/BucketPreferences.tsx index 67737f681f5..9abbd53a7a4 100644 --- a/catalog/app/components/FileEditor/QuiltConfigEditor/BucketPreferences.tsx +++ b/catalog/app/components/FileEditor/QuiltConfigEditor/BucketPreferences.tsx @@ -12,7 +12,10 @@ function Header() { return ( Configuration for Catalog UI: show and hide features, set default values. See{' '} - + the docs diff --git a/catalog/app/components/FileEditor/QuiltConfigEditor/Workflows.tsx b/catalog/app/components/FileEditor/QuiltConfigEditor/Workflows.tsx index 23175e61171..5288698cdeb 100644 --- a/catalog/app/components/FileEditor/QuiltConfigEditor/Workflows.tsx +++ b/catalog/app/components/FileEditor/QuiltConfigEditor/Workflows.tsx @@ -18,7 +18,7 @@ function Header() { return ( Configuration for data quality workflows. See{' '} - + the docs diff --git a/catalog/app/containers/Admin/Status/Status.tsx b/catalog/app/containers/Admin/Status/Status.tsx index 79cd6360655..c91d6b74d2c 100644 --- a/catalog/app/containers/Admin/Status/Status.tsx +++ b/catalog/app/containers/Admin/Status/Status.tsx @@ -54,7 +54,10 @@ export default function Status() { GxP and other compliance regimes. - + Learn more {' '} or contact sales. diff --git a/catalog/app/containers/Admin/UsersAndRoles/SsoConfig.tsx b/catalog/app/containers/Admin/UsersAndRoles/SsoConfig.tsx index b740034202e..26f292af45f 100644 --- a/catalog/app/containers/Admin/UsersAndRoles/SsoConfig.tsx +++ b/catalog/app/containers/Admin/UsersAndRoles/SsoConfig.tsx @@ -121,7 +121,10 @@ function Form({ Learn more about{' '} - + SSO permissions mapping . diff --git a/catalog/app/containers/Bucket/CodeSamples/Dir.tsx b/catalog/app/containers/Bucket/CodeSamples/Dir.tsx index bfd77c042be..5b3491691d1 100644 --- a/catalog/app/containers/Bucket/CodeSamples/Dir.tsx +++ b/catalog/app/containers/Bucket/CodeSamples/Dir.tsx @@ -14,9 +14,9 @@ const TEMPLATES = { dedent` import quilt3 as q3 b = q3.Bucket("s3://${bucket}") - # List files [[${docs}/api-reference/bucket#bucket.ls]] + # List files [[${docs}/quilt-python-sdk-developers/api-reference/bucket#bucket.ls]] b.ls("${path}") - # Download [[${docs}/api-reference/bucket#bucket.fetch]] + # Download [[${docs}/quilt-python-sdk-developers/api-reference/bucket#bucket.fetch]] b.fetch("${path}", "./${dest}") `, CLI: (bucket: string, path: string, dest: string) => diff --git a/catalog/app/containers/Bucket/CodeSamples/File.tsx b/catalog/app/containers/Bucket/CodeSamples/File.tsx index f1eea186809..006609ad393 100644 --- a/catalog/app/containers/Bucket/CodeSamples/File.tsx +++ b/catalog/app/containers/Bucket/CodeSamples/File.tsx @@ -14,7 +14,7 @@ const TEMPLATES = { dedent` import quilt3 as q3 b = q3.Bucket("s3://${bucket}") - # Download [[${docs}/api-reference/bucket#bucket.fetch]] + # Download [[${docs}/quilt-python-sdk-developers/api-reference/bucket#bucket.fetch]] b.fetch("${path}", "./${basename(path)}") `, CLI: (bucket: string, path: string) => diff --git a/catalog/app/containers/Bucket/CodeSamples/Package.tsx b/catalog/app/containers/Bucket/CodeSamples/Package.tsx index 7dbce6e97ee..c2763b3d750 100644 --- a/catalog/app/containers/Bucket/CodeSamples/Package.tsx +++ b/catalog/app/containers/Bucket/CodeSamples/Package.tsx @@ -19,16 +19,16 @@ const TEMPLATES = { const hashPy = hashDisplay && `, top_hash="${hashDisplay}"` return dedent` import quilt3 as q3 - # Browse [[${docs}/api-reference/package#package.browse]] + # Browse [[${docs}/quilt-python-sdk-developers/api-reference/package#package.browse]] p = q3.Package.browse("${name}"${hashPy}, registry="s3://${bucket}") - # make changes to package adding individual files [[${docs}/api-reference/package#package.set]] + # make changes to package adding individual files [[${docs}/quilt-python-sdk-developers/api-reference/package#package.set]] p.set("data.csv", "data.csv") - # or whole directories [[${docs}/api-reference/package#package.set_dir]] + # or whole directories [[${docs}/quilt-python-sdk-developers/api-reference/package#package.set_dir]] p.set_dir("subdir", "subdir") - # and push changes [[${docs}/api-reference/package#package.push]] + # and push changes [[${docs}/quilt-python-sdk-developers/api-reference/package#package.push]] p.push("${name}", registry="s3://${bucket}", message="Hello World") - # Download (be mindful of large packages) [[${docs}/api-reference/package#package.push]] + # Download (be mindful of large packages) [[${docs}/quilt-python-sdk-developers/api-reference/package#package.install]] q3.Package.install("${name}"${pathPy}${hashPy}, registry="s3://${bucket}", dest=".") ` }, @@ -36,13 +36,13 @@ const TEMPLATES = { const pathCli = path && ` --path "${s3paths.ensureNoSlash(path)}"` const hashCli = hashDisplay && ` --top-hash ${hashDisplay}` return dedent` - # Download package [[${docs}/api-reference/cli#install]] + # Download package [[${docs}/quilt-python-sdk-developers/api-reference/cli#install]] quilt3 install "${name}"${pathCli}${hashCli} --registry s3://${bucket} --dest . ` }, CLI_UPLOAD: (bucket: string, name: string) => dedent` - # Upload package [[${docs}/api-reference/cli#push]] + # Upload package [[${docs}/quilt-python-sdk-developers/api-reference/cli#push]] echo "Hello World" > README.md quilt3 push "${name}" --registry s3://${bucket} --dir . `, diff --git a/catalog/app/containers/Bucket/PackageDialog/DialogError.tsx b/catalog/app/containers/Bucket/PackageDialog/DialogError.tsx index 291cd6c3a75..3e04966ab87 100644 --- a/catalog/app/containers/Bucket/PackageDialog/DialogError.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/DialogError.tsx @@ -42,7 +42,7 @@ const errorDisplay = R.cond([ Please fix the{' '} workflows config{' '} according to{' '} - + the documentation . diff --git a/catalog/app/containers/Bucket/PackageDialog/SelectWorkflow.tsx b/catalog/app/containers/Bucket/PackageDialog/SelectWorkflow.tsx index 55f14548fcb..d3a1bc8ca4b 100644 --- a/catalog/app/containers/Bucket/PackageDialog/SelectWorkflow.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/SelectWorkflow.tsx @@ -71,7 +71,7 @@ export default function SelectWorkflow({ {!!error && {error}} - + Learn about data quality workflows , or edit{' '} diff --git a/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx b/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx index b9713539063..f0387219d4d 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx @@ -97,8 +97,10 @@ function WorkgroupsEmpty({ error }: WorkgroupsEmptyProps) { Check{' '} - Athena Queries docs on - setup and correct usage + + Athena Queries docs + {' '} + on setup and correct usage diff --git a/catalog/app/containers/Bucket/Successors.tsx b/catalog/app/containers/Bucket/Successors.tsx index 3a0e5449b34..5fc1a51cc40 100644 --- a/catalog/app/containers/Bucket/Successors.tsx +++ b/catalog/app/containers/Bucket/Successors.tsx @@ -28,7 +28,7 @@ function EmptySlot({ bucket }: EmptySlotProps) { Learn more @@ -52,7 +52,7 @@ function ErrorSlot({ error }: ErrorSlotProps) { {error instanceof ERRORS.WorkflowsConfigInvalid && ( Please fix the workflows config according to{' '} - + the documentation diff --git a/catalog/app/containers/Bucket/Summarize.tsx b/catalog/app/containers/Bucket/Summarize.tsx index b88512727dd..ebc2116b441 100644 --- a/catalog/app/containers/Bucket/Summarize.tsx +++ b/catalog/app/containers/Bucket/Summarize.tsx @@ -618,7 +618,9 @@ function SummaryFailed({ error }: SummaryFailedProps) { Check your quilt_summarize.json file for errors. See the{' '} - + summarize docs {' '} for more. diff --git a/catalog/app/containers/Bucket/errors.tsx b/catalog/app/containers/Bucket/errors.tsx index fd45d3399ca..5a91f621431 100644 --- a/catalog/app/containers/Bucket/errors.tsx +++ b/catalog/app/containers/Bucket/errors.tsx @@ -124,7 +124,7 @@ const defaultHandlers: ErrorHandler[] = [
Learn how to configure the bucket for Quilt @@ -167,7 +167,7 @@ const defaultHandlers: ErrorHandler[] = [
Learn about access control in Quilt diff --git a/catalog/app/containers/NavBar/Suggestions/Suggestions.tsx b/catalog/app/containers/NavBar/Suggestions/Suggestions.tsx index 4fa912fe30d..d459995f009 100644 --- a/catalog/app/containers/NavBar/Suggestions/Suggestions.tsx +++ b/catalog/app/containers/NavBar/Suggestions/Suggestions.tsx @@ -61,7 +61,10 @@ function SuggestionsList({ items, selected }: SuggestionsProps) { ))}
Learn the{' '} - + advanced search syntax {' '} for query string queries in ElasticSearch {ES_V}.