diff --git a/apps/README.md b/apps/README.md new file mode 100644 index 0000000..15c14e4 --- /dev/null +++ b/apps/README.md @@ -0,0 +1,4 @@ +# apps + +This directory contains demonstrations and experimental command-line tools +written in Dart to use the Registry API. diff --git a/apps/googleapis.sh b/apps/googleapis.sh deleted file mode 100644 index 53779b1..0000000 --- a/apps/googleapis.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -apg registry create-project \ - --project_id googleapis \ - --project.display_name "Google APIs" \ - --project.description "Google API descriptions from a variety of sources" diff --git a/apps/importer/lib/importer.dart b/apps/importer/lib/importer.dart deleted file mode 100644 index 2e396dc..0000000 --- a/apps/importer/lib/importer.dart +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'package:grpc/grpc.dart' as grpc; -import 'package:registry/registry.dart' as rpc; - -class GoogleApis { - static String idForTitle(String title) { - var id = title.toLowerCase(); - // remove the google prefix because it is applied inconsistently - if (id.startsWith("google ")) { - id = id.substring("google ".length); - } - // remove anything in parentheses - if (id.contains("(")) { - final start = id.indexOf("("); - final end = id.indexOf(")"); - id = id.substring(0, start - 1) + id.substring(end + 1); - } - // remove some superfluous words - if (id.contains(" api")) { - id = id.replaceAll(" api", ""); - } - if (id.contains(" v2")) { - id = id.replaceAll(" v2", ""); - } - if (id.contains(" v3")) { - id = id.replaceAll(" v3", ""); - } - // make it printable - id = id.replaceAll("&", "and"); - id = id.replaceAll("/", ""); - id = id.replaceAll(" ", "-"); - // consistently prefix with "google-" - id = "google-" + id; - return id; - } - - static String titleForTitle(String title) { - if (!title.startsWith("Google ")) { - title = "Google " + title; - } - return title; - } -} - -extension ImportingAdmin on rpc.AdminClient { - Future projectExists(String name) async { - try { - await this.getProject(rpc.GetProjectRequest()..name = name); - return true; - } on grpc.GrpcError catch (error) { - if (error.code == grpc.StatusCode.notFound) { - return false; - } else if (error.code == grpc.StatusCode.invalidArgument) { - return false; - } - rethrow; - } - } - - void ensureProjectExists(rpc.Project project) async { - if (!await this.projectExists(project.name)) { - try { - await this.createProject(rpc.CreateProjectRequest() - ..projectId = project.name.split("/").last - ..project = project); - } on grpc.GrpcError catch (error) { - if (error.code != grpc.StatusCode.alreadyExists) { - rethrow; - } - } - } - } -} - -extension Importing on rpc.RegistryClient { - Future apiExists(String name) async { - try { - await this.getApi(rpc.GetApiRequest()..name = name); - return true; - } on grpc.GrpcError catch (error) { - if (error.code == grpc.StatusCode.notFound) { - return false; - } - rethrow; - } - } - - Future apiVersionExists(String name) async { - try { - await this.getApiVersion(rpc.GetApiVersionRequest()..name = name); - return true; - } on grpc.GrpcError catch (error) { - if (error.code == grpc.StatusCode.notFound) { - return false; - } - rethrow; - } - } - - Future apiSpecExists(String name) async { - try { - await this.getApiSpec(rpc.GetApiSpecRequest()..name = name); - return true; - } on grpc.GrpcError catch (error) { - if (error.code == grpc.StatusCode.notFound) { - return false; - } - rethrow; - } - } - - void ensureApiExists(rpc.Api api) async { - if (!await this.apiExists(api.name)) { - try { - await this.createApi(rpc.CreateApiRequest() - ..parent = api.name.split("/").sublist(0, 2).join("/") - ..apiId = api.name.split("/").last - ..api = api); - } on grpc.GrpcError catch (error) { - if (error.code != grpc.StatusCode.alreadyExists) { - rethrow; - } - } - } - } - - void ensureApiVersionExists(rpc.ApiVersion version) async { - if (!await this.apiVersionExists(version.name)) { - try { - await this.createApiVersion(rpc.CreateApiVersionRequest() - ..parent = version.name.split("/").sublist(0, 4).join("/") - ..apiVersionId = version.name.split("/").last - ..apiVersion = version); - } on grpc.GrpcError catch (error) { - if (error.code != grpc.StatusCode.alreadyExists) { - rethrow; - } - } - } - } -} diff --git a/apps/importer/pubspec.yaml b/apps/importer/pubspec.yaml deleted file mode 100644 index e634887..0000000 --- a/apps/importer/pubspec.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: importer -description: Common support code for Registry API importers. - -# The following line prevents the package from being accidentally published to -# pub.dev using `pub publish`. This is preferred for private packages. -publish_to: 'none' # Remove this line if you wish to publish to pub.dev - -version: 1.0.0+1 - -environment: - sdk: ">=2.7.0 <3.0.0" - -dependencies: - grpc: ^3.0.0 - http: ^0.13.0 - protobuf: ^2.0.0 - \ No newline at end of file diff --git a/apps/motley.sh b/apps/motley.sh deleted file mode 100644 index 111f309..0000000 --- a/apps/motley.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -apg registry delete-project \ - --name projects/motley - -apg registry create-project \ - --project_id motley \ - --project.display_name "Motley APIs" \ - --project.description "API descriptions from a variety of sources" - -reg import common-protos --project projects/motley --path ~/Desktop/api-common-protos -reg import googleapis --project projects/motley --path ~/Desktop/googleapis -reg compute summary --project projects/motley -reg label apis --project projects/motley - -registry compute references projects/motley/apis/-/versions/-/specs/- -registry compute complexity projects/motley/apis/-/versions/-/specs/- -registry compute index projects/motley/apis/-/versions/-/specs/- -registry compute vocabulary projects/motley/apis/-/versions/-/specs/- -registry compute lint projects/motley/apis/-/versions/-/specs/- -registry compute lintstats projects/motley/apis/-/versions/-/specs/- --linter=aip - diff --git a/apps/openapi.sh b/apps/openapi.sh deleted file mode 100644 index c230838..0000000 --- a/apps/openapi.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -apg registry create-project \ - --project_id openapi \ - --project.display_name "OpenAPI Directory" \ - --project.description "API descriptions from the OpenAPI Directory" diff --git a/apps/reg/README.md b/apps/reg/README.md new file mode 100644 index 0000000..1f12a06 --- /dev/null +++ b/apps/reg/README.md @@ -0,0 +1,5 @@ +# reg + +This directory contains `reg`, a small Dart command-line tool for +working with the Registry API. `reg` provides a central invocation +point for a collection of random demonstrations and experiments. diff --git a/apps/reg/lib/src/import-asyncapi-directory.dart b/apps/reg/lib/src/import-asyncapi-directory.dart deleted file mode 100644 index dd6953f..0000000 --- a/apps/reg/lib/src/import-asyncapi-directory.dart +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'dart:convert'; -import 'dart:io'; - -import 'package:archive/archive.dart'; -import 'package:args/command_runner.dart'; -import 'package:importer/importer.dart'; -import 'package:registry/registry.dart' as rpc; -import 'package:yaml/yaml.dart'; - -final source = "asyncapi_directory"; - -final maxDescriptionLength = 140; - -class ImportAsyncAPIDirectoryCommand extends Command { - final name = "asyncapi-directory"; - final description = "Import specs from an AsyncAPI Directory."; - - ImportAsyncAPIDirectoryCommand() { - this.argParser - ..addOption( - 'project', - help: "Project for imports.", - valueHelp: "PROJECT", - ) - ..addOption( - 'path', - help: "Path to a directory containing AsyncAPI descriptions.", - valueHelp: "PATH", - ); - } - - void run() async { - if (argResults['project'] == null) { - throw UsageException("Please specify --project", this.argParser.usage); - } - if (argResults['path'] == null) { - throw UsageException("Please specify --path", this.argParser.usage); - } - final channel = rpc.createClientChannel(); - final client = rpc.RegistryClient(channel, options: rpc.callOptions()); - final adminClient = rpc.AdminClient(channel, options: rpc.callOptions()); - - final projectName = argResults['project']; - final root = argResults['path']; - - final exists = await adminClient.projectExists(projectName); - if (!exists) { - throw UsageException("$projectName does not exist", this.argParser.usage); - } - - // API specs are in files named "asyncapi.yaml". - RegExp apiSpecPattern = new RegExp(r"/asyncapi.yaml$"); - var paths = Directory(root).listSync(recursive: true); - paths.sort((a, b) => a.path.compareTo(b.path)); - await Future.forEach(paths, (entity) async { - if (entity is File) { - if (apiSpecPattern.hasMatch(entity.path)) { - await client.importAsyncAPI(root, projectName, entity.path); - } - } - }); - - await channel.shutdown(); - } -} - -extension AsyncAPIImporter on rpc.RegistryClient { - void importAsyncAPI(String root, projectName, path) async { - var name = path.substring(root.length + 1); - - var parts = name.split("/"); - var apiParts = parts.sublist(0, parts.length - 2); - var owner = filterOwner(apiParts[0]); - var apiId; - if (apiParts.length == 1) { - apiId = owner; - } else { - apiId = owner + "-" + apiParts.sublist(1).join("-"); - } - var versionId = parts[parts.length - 2]; - var specId = parts.last; - - try { - String contents = await File(path).readAsString(); - var doc = loadYaml(contents); - - // compute mime type from internal format string - var mimeType; - var asyncapi = doc["asyncapi"]; - if (asyncapi != null) { - mimeType = "application/x.asyncapi+gzip;version=" + asyncapi; - } - var info = doc["info"]; - var apiTitle = info["title"] ?? ""; - var description = info["description"] ?? ""; - if (description.length > maxDescriptionLength) { - description = description.substring(0, maxDescriptionLength) + "..."; - } - - print("uploading $apiId $versionId $specId"); - print("$apiTitle"); - print("$description"); - - final apiName = projectName + "/apis/" + apiId; - var api = rpc.Api() - ..name = apiName - ..displayName = apiTitle - ..description = description; - api.labels["created_from"] = source; - api.labels["owner"] = owner; - await this.ensureApiExists(api); - - final versionName = apiName + "/versions/" + versionId; - var version = rpc.ApiVersion() - ..name = versionName - ..displayName = versionId; - version.labels["created_from"] = source; - await this.ensureApiVersionExists(version); - - final specName = versionName + "/specs/" + specId; - if (!await this.apiSpecExists(specName)) { - var apiSpec = rpc.ApiSpec() - ..filename = specId - ..contents = GZipEncoder().encode(Utf8Encoder().convert(contents)) - ..mimeType = mimeType; - apiSpec.labels["created_from"] = source; - var request = rpc.CreateApiSpecRequest() - ..parent = versionName - ..apiSpecId = specId - ..apiSpec = apiSpec; - await this.createApiSpec(request); - } - - // that's all - } catch (error) { - print("$error"); - } - } - - String filterOwner(String owner) { - switch (owner) { - case "googleapis.com": - return "google"; - default: - return owner; - } - } -} diff --git a/apps/reg/lib/src/import-common-protos.dart b/apps/reg/lib/src/import-common-protos.dart deleted file mode 100644 index 81ba562..0000000 --- a/apps/reg/lib/src/import-common-protos.dart +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'dart:collection'; -import 'dart:io'; - -import 'package:archive/archive.dart'; -import 'package:args/command_runner.dart'; -import 'package:grpc/grpc.dart' as grpc; -import 'package:importer/importer.dart'; -import 'package:registry/registry.dart' as rpc; - -final source = "google-common-protos"; - -class Entry { - String api; - String version; - String path; - String title; - String description; -} - -class ImportCommonProtosCommand extends Command { - final name = "common-protos"; - final description = "Import specs from the Google common protos repository."; - - ImportCommonProtosCommand() { - this.argParser - ..addOption( - 'project', - help: "Project id for imports.", - valueHelp: "PROJECT", - ) - ..addOption( - 'path', - help: "Path to a directory containing the common protos repository.", - valueHelp: "PATH", - ); - } - - void run() async { - if (argResults['project'] == null) { - throw UsageException("Please specify --project", this.argParser.usage); - } - if (argResults['path'] == null) { - throw UsageException("Please specify --path", this.argParser.usage); - } - final channel = rpc.createClientChannel(); - final client = rpc.RegistryClient(channel, options: rpc.callOptions()); - final adminClient = rpc.AdminClient(channel, options: rpc.callOptions()); - - final projectName = argResults['project']; - - final exists = await adminClient.projectExists(projectName); - await channel.shutdown(); - if (!exists) { - throw UsageException("$projectName does not exist", this.argParser.usage); - } - - String root = argResults['path']; - List entries = [ - Entry() - ..api = "googleapis.com-api" - ..version = "v1" - ..path = 'google/api' - ..title = "Google API Protos" - ..description = - "Also known as 'service config', the schema for configuration of " - "Google's internal API platform, which handles routing, quotas, " - "monitoring, logging, and the like.", - Entry() - ..api = "googleapis.com-iam" - ..version = "v1" - ..path = "google/iam/v1" - ..title = "Google IAM API" - ..description = "The Identity and Access Management (IAM) API. " - "Manages identity and access control for Google Cloud Platform " - "resources, including the creation of service accounts.", - Entry() - ..api = "googleapis.com-iam-admin" - ..version = "v1" - ..path = "google/iam/admin/v1" - ..title = "Google IAM Admin API" - ..description = - "Administration API for Identity and Access Management.", - Entry() - ..api = "googleapis.com-longrunning" - ..version = "v1" - ..path = "google/longrunning" - ..title = "Google Long-running Operations API" - ..description = "An abstract interface that manages long running " - "operations with API services.", - Entry() - ..api = "googleapis.com-logging" - ..version = "v1" - ..path = "google/logging" - ..title = "Google Logging Types" - ..description = - "Shared types popul;ated by the Stackdriver Logging API and " - "consumed by other APIs.", - Entry() - ..api = "googleapis.com-rpc" - ..version = "v1" - ..path = "google/rpc" - ..title = "Google RPC (Remote Procedure Call) Types" - ..description = "Types that represent remote procedure call concepts.", - Entry() - ..api = "googleapis.com-type" - ..version = "v1" - ..path = "google/type" - ..title = "Google Common Types" - ..description = "Common types for Google APIs.", - Entry() - ..api = "googleapis.com-protobuf" - ..version = "v1" - ..path = "google/protobuf" - ..title = "Google Protobuf Types" - ..description = - "Standard types distributed with Google's Protocol Buffers tools.", - ]; - - final Queue tasks = Queue(); - for (var entry in entries) { - tasks.add(ImportCommonProtosTask(root, projectName, entry)); - } - await rpc.TaskProcessor(tasks, 1).run(); - } -} - -class ImportCommonProtosTask implements rpc.Task { - final String root; - final String projectName; - final Entry entry; - ImportCommonProtosTask(this.root, this.projectName, this.entry); - - String name() => entry.api + " " + entry.version + " " + entry.path; - - void run(rpc.RegistryClient client) async { - await client.importProtobufApi(root, projectName, entry); - } -} - -extension GoogleApiImporter on rpc.RegistryClient { - void importProtobufApi( - String root, - String projectName, - Entry entry, - ) async { - var apiId = entry.api; - var versionId = entry.version; - - var apiTitle = entry.title; - print("apiId: " + apiId); - - var path = entry.path; - var specId = path.replaceAll("/", "-") + ".zip"; - print("specId: " + specId); - - String description = entry.description; - - final apiName = projectName + "/apis/" + apiId; - var api = rpc.Api() - ..name = apiName - ..displayName = apiTitle - ..description = description; - api.labels["created_from"] = source; - api.labels["owner"] = "googleapis.com"; - await this.ensureApiExists(api); - - final versionName = apiName + "/versions/" + versionId; - var version = rpc.ApiVersion() - ..name = versionName - ..displayName = versionId; - version.labels["created_from"] = source; - await this.ensureApiVersionExists(version); - - final dir = Directory(root + "/" + path); - final specName = versionName + "/specs/" + specId; - if (!await this.apiSpecExists(specName)) { - var archive = Archive(); - await Future.forEach(dir.listSync(recursive: true), (file) async { - if (file is File) { - var name = file.path.substring(root.length + 1); - List content = await File(file.path).readAsBytes(); - archive.addFile(ArchiveFile(name, content.length, content)); - } - }); - var apiSpec = rpc.ApiSpec() - ..filename = specId - ..contents = ZipEncoder().encode(archive) - ..sourceUri = "" - ..mimeType = "application/x.protobuf+zip"; - apiSpec.labels["created_from"] = source; - var request = rpc.CreateApiSpecRequest() - ..parent = versionName - ..apiSpecId = specId - ..apiSpec = apiSpec; - try { - await this.createApiSpec(request, options: rpc.callOptions()); - } on grpc.GrpcError catch (error) { - if (error.code != grpc.StatusCode.alreadyExists) { - rethrow; - } - } - } - } -} diff --git a/apps/reg/lib/src/import-discovery.dart b/apps/reg/lib/src/import-discovery.dart deleted file mode 100644 index b9a8485..0000000 --- a/apps/reg/lib/src/import-discovery.dart +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'dart:collection'; -import 'dart:convert'; - -import 'package:archive/archive.dart'; -import 'package:args/command_runner.dart'; -import 'package:http/http.dart' as http; -import 'package:importer/importer.dart'; -import 'package:registry/registry.dart' as rpc; - -final source = "discovery"; - -class ImportDiscoveryCommand extends Command { - final name = "discovery"; - final description = "Import specs from the Google API Discovery Service."; - - ImportDiscoveryCommand() { - this.argParser - ..addOption( - 'project', - help: "Project for imports.", - valueHelp: "PROJECT", - ); - } - - void run() async { - if (argResults['project'] == null) { - throw UsageException("Please specify --project", this.argParser.usage); - } - - final channel = rpc.createClientChannel(); - final client = rpc.RegistryClient(channel, options: rpc.callOptions()); - final adminClient = rpc.AdminClient(channel, options: rpc.callOptions()); - - final projectName = argResults['project']; - - final exists = await adminClient.projectExists(projectName); - await channel.shutdown(); - if (!exists) { - throw UsageException("$projectName does not exist", this.argParser.usage); - } - - final Queue tasks = Queue(); - - for (var item in await fetchApiListings()) { - tasks.add(ImportDiscoveryTask(item, projectName)); - } - - await rpc.TaskProcessor(tasks, 64).run(); - } -} - -Future> fetchApiListings() { - return http - .get(Uri.parse('https://www.googleapis.com/discovery/v1/apis')) - .then((response) { - Map discoveryList = jsonDecode(response.body); - return discoveryList["items"]; - }); -} - -class ImportDiscoveryTask implements rpc.Task { - final item; - final projectName; - - ImportDiscoveryTask(this.item, this.projectName); - - String name() { - final map = item as Map; - return map["name"] + " " + map["version"]; - } - - void run(rpc.RegistryClient client) async { - await client.importDiscoveryAPI(item, projectName); - } -} - -String filterOwner(String owner) { - owner = (owner ?? "Google").toLowerCase(); - if (owner == "google") { - owner = "googleapis.com"; - } - return owner; -} - -extension DiscoveryImporter on rpc.RegistryClient { - void importDiscoveryAPI(item, projectName) async { - // get basic API attributes from the Discovery Service list - var dict = item as Map; - var title = dict["title"]; - var apiId = filterOwner(dict["owner"]) + "-" + dict["name"].toLowerCase(); - var apiTitle = GoogleApis.titleForTitle(title); - var versionId = dict["version"] as String; - - // read the discovery doc - var discoveryUrl = dict["discoveryRestUrl"] as String; - var doc = await http.get(Uri.parse(discoveryUrl)); - Map discoveryDoc = jsonDecode(doc.body); - var description = discoveryDoc["description"] ?? ""; - - final apiName = projectName + "/apis/" + apiId; - var api = rpc.Api() - ..name = apiName - ..displayName = apiTitle - ..description = description; - api.labels["created_from"] = source; - api.labels["owner"] = "googleapis.com"; - await this.ensureApiExists(api); - - final versionName = apiName + "/versions/" + versionId; - var version = rpc.ApiVersion() - ..name = versionName - ..displayName = versionId; - version.labels["created_from"] = source; - await this.ensureApiVersionExists(version); - - final specName = versionName + "/specs/discovery.json"; - if (!await this.apiSpecExists(specName)) { - try { - var contents = GZipEncoder().encode(doc.bodyBytes); - var apiSpec = rpc.ApiSpec() - ..filename = "discovery.json" - ..contents = contents - ..sourceUri = discoveryUrl - ..mimeType = "application/x.discovery+gzip"; - apiSpec.labels["created_from"] = source; - var revision = discoveryDoc["revision"]; - if (revision != null) { - apiSpec.labels["revision-date"] = revision; - } - var request = rpc.CreateApiSpecRequest() - ..parent = versionName - ..apiSpecId = specName.split("/").last - ..apiSpec = apiSpec; - await this.createApiSpec(request); - } catch (error) { - print("$error"); - print(discoveryUrl); - print(doc.body.toString()); - } - } - } -} diff --git a/apps/reg/lib/src/import-googleapis.dart b/apps/reg/lib/src/import-googleapis.dart deleted file mode 100644 index eb93406..0000000 --- a/apps/reg/lib/src/import-googleapis.dart +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'dart:collection'; -import 'dart:io'; - -import 'package:archive/archive.dart'; -import 'package:args/command_runner.dart'; -import 'package:grpc/grpc.dart' as grpc; -import 'package:importer/importer.dart'; -import 'package:registry/registry.dart' as rpc; -import 'package:yaml/yaml.dart'; - -final source = "googleapis"; - -class ImportGoogleAPIsCommand extends Command { - final name = "googleapis"; - final description = "Import specs from the GoogleAPIs repository."; - - ImportGoogleAPIsCommand() { - this.argParser - ..addOption( - 'project', - help: "Project for imports.", - valueHelp: "PROJECT", - ) - ..addOption( - 'path', - help: "Path to a directory containing the GoogleAPIs repository.", - valueHelp: "PATH", - ); - } - - void run() async { - if (argResults['project'] == null) { - throw UsageException("Please specify --project", this.argParser.usage); - } - if (argResults['path'] == null) { - throw UsageException("Please specify --path", this.argParser.usage); - } - final channel = rpc.createClientChannel(); - final client = rpc.RegistryClient(channel, options: rpc.callOptions()); - final adminClient = rpc.AdminClient(channel, options: rpc.callOptions()); - - final projectName = argResults['project']; - - final exists = await adminClient.projectExists(projectName); - await channel.shutdown(); - if (!exists) { - throw UsageException("$projectName does not exist", this.argParser.usage); - } - - String root = argResults['path']; - - // API versions are in directories with names ending with this pattern. - RegExp versionDirectoryPattern = - new RegExp(r"/(v(\d+)((alpha|beta)\d+)?)$"); - final Queue tasks = Queue(); - await Future.forEach(Directory(root).listSync(recursive: true), - (entity) async { - if (entity is Directory) { - final match = versionDirectoryPattern.firstMatch(entity.path); - if ((match != null) && (match.group(1) != null)) { - tasks.add( - ImportGoogleApiTask(root, projectName, entity, match.group(1))); - } - } - }); - await rpc.TaskProcessor(tasks, 64).run(); - } -} - -class ImportGoogleApiTask implements rpc.Task { - final String root; - final String projectName; - final Directory entity; - final String versionId; - ImportGoogleApiTask(this.root, this.projectName, this.entity, this.versionId); - - String name() => entity.path + " " + versionId; - - void run(rpc.RegistryClient client) async { - await client.importProtobufApi(root, projectName, entity, versionId); - } -} - -extension GoogleApiImporter on rpc.RegistryClient { - void importProtobufApi( - String root, String projectName, Directory dir, String versionId) async { - // only import APIs with service yaml that specifies their title and name - RegExp yamlPattern = new RegExp(r"\.yaml$"); - await Future.forEach(dir.listSync(recursive: true), (entity) async { - if ((entity is File) && - yamlPattern.hasMatch(entity.path) && - !entity.path.contains("gapic")) { - String contents = await File(entity.path).readAsString(); - var doc = loadYaml(contents); - if ((doc["type"] == "google.api.Service") && - (doc["title"] != null) && - (doc["name"] != null)) { - var apiTitle = GoogleApis.titleForTitle(doc["title"]); - - var apiId = doc["name"]; - String suffix = ".googleapis.com"; - if (apiId.endsWith(suffix)) { - apiId = apiId.substring(0, apiId.length - suffix.length); - apiId = "googleapis.com-" + apiId; - } - print("apiId: " + apiId); - - var path = dir.path.substring(root.length + 1); - var specId = path.replaceAll("/", "-") + ".zip"; - print("specId: " + specId); - - String description = ""; - var documentation = doc["documentation"]; - if (documentation != null) { - var summary = documentation["summary"]; - if (summary != null) { - description = summary as String; - description = description.replaceAll("\n", " "); - } - } - final apiName = projectName + "/apis/" + apiId; - var api = rpc.Api() - ..name = apiName - ..displayName = apiTitle - ..description = description; - api.labels["created_from"] = source; - api.labels["owner"] = "googleapis.com"; - await this.ensureApiExists(api); - - final versionName = apiName + "/versions/" + versionId; - var version = rpc.ApiVersion() - ..name = versionName - ..displayName = versionId; - version.labels["created_from"] = source; - await this.ensureApiVersionExists(version); - - final specName = versionName + "/specs/" + specId; - if (!await this.apiSpecExists(specName)) { - var archive = Archive(); - await Future.forEach(dir.listSync(recursive: false), (file) async { - if (file is File) { - var name = file.path.substring(root.length + 1); - List content = await File(file.path).readAsBytes(); - archive.addFile(ArchiveFile(name, content.length, content)); - } - }); - var apiSpec = rpc.ApiSpec() - ..filename = specId - ..contents = ZipEncoder().encode(archive) - ..sourceUri = "" - ..mimeType = "application/x.protobuf+zip"; - apiSpec.labels["created_from"] = source; - var request = rpc.CreateApiSpecRequest() - ..parent = versionName - ..apiSpecId = specId - ..apiSpec = apiSpec; - try { - await this.createApiSpec(request, options: rpc.callOptions()); - } on grpc.GrpcError catch (error) { - if (error.code != grpc.StatusCode.alreadyExists) { - rethrow; - } - } - } - } - } - }); - } -} diff --git a/apps/reg/lib/src/import-openapi-directory.dart b/apps/reg/lib/src/import-openapi-directory.dart deleted file mode 100644 index 622c8e9..0000000 --- a/apps/reg/lib/src/import-openapi-directory.dart +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'dart:collection'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:archive/archive.dart'; -import 'package:args/command_runner.dart'; -import 'package:importer/importer.dart'; -import 'package:registry/registry.dart' as rpc; -import 'package:yaml/yaml.dart'; - -final source = "openapi_directory"; - -final maxDescriptionLength = 140; - -class ImportOpenAPIDirectoryCommand extends Command { - final name = "openapi-directory"; - final description = "Import specs from the OpenAPI Directory."; - - ImportOpenAPIDirectoryCommand() { - this.argParser - ..addOption( - 'project', - help: "Project for imports.", - valueHelp: "PROJECT", - ) - ..addOption( - 'path', - help: "Path to a directory containing OpenAPI descriptions.", - valueHelp: "PATH", - ); - } - - void run() async { - if (argResults['project'] == null) { - throw UsageException("Please specify --project", this.argParser.usage); - } - if (argResults['path'] == null) { - throw UsageException("Please specify --path", this.argParser.usage); - } - final channel = rpc.createClientChannel(); - final client = rpc.RegistryClient(channel, options: rpc.callOptions()); - final adminClient = rpc.AdminClient(channel, options: rpc.callOptions()); - - final projectName = argResults['project']; - final root = argResults['path']; - - final exists = await adminClient.projectExists(projectName); - await channel.shutdown(); - if (!exists) { - throw UsageException("$projectName does not exist", this.argParser.usage); - } - - final Queue tasks = Queue(); - - // API specs are in files named "swagger.yaml" or "openapi.yaml". - RegExp apiSpecPattern = new RegExp(r"/(swagger|openapi).yaml$"); - var paths = Directory(root).listSync(recursive: true); - paths.sort((a, b) => a.path.compareTo(b.path)); - await Future.forEach(paths, (entity) async { - if (entity is File) { - if (apiSpecPattern.hasMatch(entity.path)) { - tasks.add(ImportOpenAPIDirectoryTask(root, entity.path, projectName)); - } - } - }); - - await rpc.TaskProcessor(tasks, 64).run(); - } -} - -class ImportOpenAPIDirectoryTask implements rpc.Task { - final String root; - final String path; - final String projectName; - - ImportOpenAPIDirectoryTask(this.root, this.path, this.projectName); - - String name() => path; - - void run(rpc.RegistryClient client) async { - var name = path.substring(root.length + 1); - - var parts = name.split("/"); - var apiParts = parts.sublist(0, parts.length - 2); - var owner = filterOwner(apiParts[0]); - var apiId; - if (apiParts.length == 1) { - apiId = owner; - } else { - apiId = owner + "-" + apiParts.sublist(1).join("-"); - } - apiId = apiId.toLowerCase(); - var versionId = parts[parts.length - 2]; - var specId = parts.last; - - try { - String contents = await File(path).readAsString(); - var doc = loadYaml(contents); - - // compute mime type from internal format string - var mimeType; - var swagger = doc["swagger"]; - if (swagger != null) { - mimeType = "application/x.openapi+gzip;version=" + swagger; - } - var openapi = doc["openapi"]; - if (openapi != null) { - mimeType = "application/x.openapi+gzip;version=" + openapi; - } - var info = doc["info"]; - var apiTitle = info["title"] ?? ""; - if (owner == "google") { - apiTitle = GoogleApis.titleForTitle(apiTitle); - } - var description = info["description"] ?? ""; - if (description.length > maxDescriptionLength) { - description = description.substring(0, maxDescriptionLength) + "..."; - } - - final apiName = projectName + "/apis/" + apiId; - var api = rpc.Api() - ..name = apiName - ..displayName = apiTitle - ..description = description; - api.labels["created_from"] = source; - api.labels["owner"] = owner; - await client.ensureApiExists(api); - - final versionName = apiName + "/versions/" + versionId; - var version = rpc.ApiVersion() - ..name = versionName - ..displayName = versionId; - version.labels["created_from"] = source; - await client.ensureApiVersionExists(version); - - final specName = versionName + "/specs/" + specId; - if (!await client.apiSpecExists(specName)) { - var apiSpec = rpc.ApiSpec() - ..filename = specId - ..contents = GZipEncoder().encode(Utf8Encoder().convert(contents)) - ..mimeType = mimeType; - apiSpec.labels["created_from"] = source; - var request = rpc.CreateApiSpecRequest() - ..parent = versionName - ..apiSpecId = specId - ..apiSpec = apiSpec; - await client.createApiSpec(request); - } - } catch (error) { - print("$error"); - } - } - - String filterOwner(String owner) { - return owner.toLowerCase(); - } -} diff --git a/apps/reg/lib/src/import.dart b/apps/reg/lib/src/import.dart deleted file mode 100644 index ed9889a..0000000 --- a/apps/reg/lib/src/import.dart +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2021 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import 'package:args/command_runner.dart'; -import 'import-asyncapi-directory.dart'; -import 'import-common-protos.dart'; -import 'import-discovery.dart'; -import 'import-googleapis.dart'; -import 'import-openapi-directory.dart'; - -class ImportCommand extends Command { - final name = "import"; - final description = "Import API descriptions."; - - ImportCommand() { - this.addSubcommand(ImportAsyncAPIDirectoryCommand()); - this.addSubcommand(ImportCommonProtosCommand()); - this.addSubcommand(ImportDiscoveryCommand()); - this.addSubcommand(ImportGoogleAPIsCommand()); - this.addSubcommand(ImportOpenAPIDirectoryCommand()); - } -} diff --git a/apps/reg/lib/src/label-apis.dart b/apps/reg/lib/src/label-apis.dart index 355613b..1038f53 100644 --- a/apps/reg/lib/src/label-apis.dart +++ b/apps/reg/lib/src/label-apis.dart @@ -31,11 +31,11 @@ class LabelAPIsCommand extends Command { } void run() async { - if (argResults['project'] == null) { + if (argResults!['project'] == null) { throw UsageException("Please specify --project", this.argParser.usage); } - final projectName = argResults['project']; + final projectName = argResults!['project']; final channel = rpc.createClientChannel(); final client = rpc.RegistryClient(channel, options: rpc.callOptions()); @@ -52,11 +52,11 @@ class LabelAPIsCommand extends Command { await channel.shutdown(); - await rpc.TaskProcessor(tasks, 64).run(); + await rpc.TaskProcessor(tasks, 4).run(); } } -String typeFromMimeType(String mimeType) { +String? typeFromMimeType(String mimeType) { RegExp mimeTypePattern = new RegExp(r"^application/x.([^\+;]*)(.*)?$"); var match = mimeTypePattern.firstMatch(mimeType); if (match != null) { @@ -66,24 +66,24 @@ String typeFromMimeType(String mimeType) { } class LabelApiTask implements rpc.Task { - final String apiName; + final String? apiName; LabelApiTask(this.apiName); - String name() => apiName; + String name() => apiName!; - void run(rpc.RegistryClient client) async { - var getRequest = rpc.GetApiRequest()..name = apiName; + Future run(rpc.RegistryClient client) async { + var getRequest = rpc.GetApiRequest()..name = apiName!; rpc.Api api = await client.getApi(getRequest); int versionCount = 0; int specCount = 0; - Map apiSpecTypes = {}; + Map apiSpecTypes = {}; await rpc.listAPIVersions( client, - parent: apiName, + parent: apiName!, f: (version) async { versionCount++; - Map versionSpecTypes = {}; + Map versionSpecTypes = {}; await rpc.listAPISpecs(client, parent: version.name, f: (spec) { specCount++; var type = typeFromMimeType(spec.mimeType); @@ -102,7 +102,7 @@ class LabelApiTask implements rpc.Task { api.labels["versions"] = "$versionCount"; api.labels["specs"] = "$specCount"; for (var key in apiSpecTypes.keys) { - api.labels[key] = "true"; + api.labels[key!] = "true"; } var updateRequest = rpc.UpdateApiRequest() ..api = api diff --git a/apps/reg/lib/src/root.dart b/apps/reg/lib/src/root.dart index 59e51ab..b7b6f91 100644 --- a/apps/reg/lib/src/root.dart +++ b/apps/reg/lib/src/root.dart @@ -14,13 +14,11 @@ import 'package:args/command_runner.dart'; -import 'import.dart'; import 'label.dart'; import 'scan.dart'; CommandRunner root() { return CommandRunner("reg", "Another API Registry tool.") - ..addCommand(ImportCommand()) ..addCommand(LabelCommand()) ..addCommand(ScanCommand()); } diff --git a/apps/reg/lib/src/scan-security.dart b/apps/reg/lib/src/scan-security.dart index 90033a7..667ecbe 100644 --- a/apps/reg/lib/src/scan-security.dart +++ b/apps/reg/lib/src/scan-security.dart @@ -18,8 +18,6 @@ import 'package:args/command_runner.dart'; import 'package:registry/registry.dart' as rpc; import 'package:yaml/yaml.dart'; -final source = "discovery"; - class ScanSecurityCommand extends Command { final name = "security"; final description = "Scan security fields in OpenAPI specs."; @@ -39,12 +37,12 @@ class ScanSecurityCommand extends Command { } void run() async { - if (argResults['version'] == null) { + if (argResults!['version'] == null) { throw UsageException("Please specify --version", this.argParser.usage); } - String versionName = argResults['version']; - String filter = argResults['filter'] ?? ""; + String versionName = argResults!['version']; + String filter = argResults!['filter'] ?? ""; final channel = rpc.createClientChannel(); final client = rpc.RegistryClient(channel, options: rpc.callOptions()); @@ -75,7 +73,7 @@ class ScanSecurityCommand extends Command { if (doc["openapi"] != null) { // look for openapi v3 security var security = doc["security"]; - var securitySchemes = null; + dynamic securitySchemes = null; if (doc["components"] != null) { securitySchemes = doc["components"]["securitySchemes"]; } @@ -98,7 +96,7 @@ class ScanSecurityCommand extends Command { } } -String typeFromMimeType(String mimeType) { +String? typeFromMimeType(String mimeType) { RegExp mimeTypePattern = new RegExp(r"^application/x.([^\+;]*)(.*)?$"); var match = mimeTypePattern.firstMatch(mimeType); if (match != null) { diff --git a/apps/reg/pubspec.yaml b/apps/reg/pubspec.yaml index fe5be2c..9897ec2 100644 --- a/apps/reg/pubspec.yaml +++ b/apps/reg/pubspec.yaml @@ -2,11 +2,9 @@ name: reg description: Manage information in an API Registry. environment: - sdk: '>=2.10.0 <3.0.0' + sdk: '>=2.17.0 <3.0.0' dependencies: - importer: - path: ../importer registry: path: ../../registry archive: ^3.0.0 diff --git a/registry/lib/src/list.dart b/registry/lib/src/list.dart index 23f38e7..ade75f2 100644 --- a/registry/lib/src/list.dart +++ b/registry/lib/src/list.dart @@ -18,7 +18,7 @@ import 'grpc_client.dart'; void nil() {} -void listProjects(AdminClient client, +Future listProjects(AdminClient client, {Function f = nil, String filter = ''}) async { var request = ListProjectsRequest() ..filter = filter @@ -35,7 +35,7 @@ void listProjects(AdminClient client, } } -void listAPIs(RegistryClient client, +Future listAPIs(RegistryClient client, {Function f = nil, String parent = '', String filter = ''}) async { var request = ListApisRequest() ..parent = parent @@ -53,7 +53,7 @@ void listAPIs(RegistryClient client, } } -void listAPIVersions(RegistryClient client, +Future listAPIVersions(RegistryClient client, {Function f = nil, String parent = '', String filter = ''}) async { var request = ListApiVersionsRequest() ..parent = parent @@ -72,7 +72,7 @@ void listAPIVersions(RegistryClient client, } } -void listAPISpecs(RegistryClient client, +Future listAPISpecs(RegistryClient client, {Function f = nil, String parent = '', String filter = ''}) async { var request = ListApiSpecsRequest() ..parent = parent diff --git a/registry/lib/src/tasks.dart b/registry/lib/src/tasks.dart index 90eff6f..0e027a8 100644 --- a/registry/lib/src/tasks.dart +++ b/registry/lib/src/tasks.dart @@ -20,7 +20,7 @@ import 'grpc_client.dart'; import 'generated/google/cloud/apigeeregistry/v1/registry_service.pbgrpc.dart'; abstract class Task { - void run(RegistryClient client); + Future run(RegistryClient client); String name(); } @@ -29,7 +29,7 @@ class TaskProcessor { final int width; TaskProcessor(this.queue, this.width); - void run() async { + Future run() async { var futures = []; for (var i = 0; i < width; i++) { futures.add(startWorker(i, queue)); @@ -90,7 +90,7 @@ class TaskProcessor { } else if (data is Task) { print('->[$id] ${data.name()}'); try { - data.run(client); + await data.run(client); } catch (error) { print("$error"); }