Connector Specification
DNAStack Publisher Connector SPI Version 0.1.0
The internal Service Provider Interface implemented by DNAstack Publisher Connector services.
get /connection-types
Lists the connection-types
Description: Lists the types of connections supported by this connector
Security: dlcon:connection-type:read
Response Code: 200
Description: A list of connection-types
Schema:
{
"properties": {
"icon": {
"description": "URL location icon for this kind of connection",
"example": "https://storage.googleapis.com/data-lake/icons/azure-blobstore.jpg",
"type": "string"
},
"id": {
"description": "Connection Type",
"example": "azure-blobstore",
"type": "string"
}
},
"required": [
"id",
"icon"
],
"type": "object"
}get /connections
Lists connections
Description: Lists existing connections
Security: dlcon:connection:list
Response Code: 200
Description: A list of connections
Schema:
{
"properties": {
"configuration": {
"description": "Configuration special to the connector type.\n\nThe exact content of this object vary based on the needs of the backend API the connector implementaton is designed to connect to.\nTypically includes network coordinates (eg. URL, hostname, port number, proxy configuraion) and credentials (eg. service account\nkeys or username / password) for accessing the underlying resource. Nothing outside this connector implementation needs to\nunderstand the contents of this object.\n\nImplementors should be careful not to include secrets when sending out a connector configuration (secrets should be treated as\nwrite-only properties).",
"type": "object"
},
"createdAt": {
"description": "The RFC3339 date and time when this connection was created.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"createdBy": {
"description": "The user that created the connection.",
"type": "string"
},
"id": {
"description": "Unique connection identifier",
"example": "615e48e5-2cf9-4968-95cd-2a61013b563b",
"format": "uuid",
"type": "string"
},
"type": {
"description": "The type of this connection. Must match the id of a ConnectionType supported by this connector service.",
"example": "azure-blobstore",
"type": "string"
},
"updatedAt": {
"description": "The RFC3339 date and time when the connection was last updated.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"updatedBy": {
"description": "The user that last updated the connection.",
"type": "string"
},
"version": {
"description": "Read-only property that changes every time this connection object is modified. Also returned in the eTag header when GETting the\nconnection. This value is required in the If-Match header of PATCH requests to support optimistic locking.",
"type": "string"
}
},
"required": [
"id",
"configuration",
"type",
"createdAt",
"createdBy"
],
"type": "object"
}Response Code: 401
Description: Authentication Required
Response Code: 500
Description: Internal Server Error
post /connections
Creates a connection to a data source
Description: Creates a new connection instance
Security: dlcon:connection:write
Response Code: 200
Description: Connector connection
Schema:
{
"properties": {
"configuration": {
"description": "Configuration special to the connector type.\n\nThe exact content of this object vary based on the needs of the backend API the connector implementaton is designed to connect to.\nTypically includes network coordinates (eg. URL, hostname, port number, proxy configuraion) and credentials (eg. service account\nkeys or username / password) for accessing the underlying resource. Nothing outside this connector implementation needs to\nunderstand the contents of this object.\n\nImplementors should be careful not to include secrets when sending out a connector configuration (secrets should be treated as\nwrite-only properties).",
"type": "object"
},
"createdAt": {
"description": "The RFC3339 date and time when this connection was created.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"createdBy": {
"description": "The user that created the connection.",
"type": "string"
},
"id": {
"description": "Unique connection identifier",
"example": "615e48e5-2cf9-4968-95cd-2a61013b563b",
"format": "uuid",
"type": "string"
},
"type": {
"description": "The type of this connection. Must match the id of a ConnectionType supported by this connector service.",
"example": "azure-blobstore",
"type": "string"
},
"updatedAt": {
"description": "The RFC3339 date and time when the connection was last updated.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"updatedBy": {
"description": "The user that last updated the connection.",
"type": "string"
},
"version": {
"description": "Read-only property that changes every time this connection object is modified. Also returned in the eTag header when GETting the\nconnection. This value is required in the If-Match header of PATCH requests to support optimistic locking.",
"type": "string"
}
},
"required": [
"id",
"configuration",
"type",
"createdAt",
"createdBy"
],
"type": "object"
}Response Code: 401
Description: Authentication Required
Response Code: 409
Description: Conflict
Response Code: 500
Description: Internal Server Error
delete /connections/{connectionId}
Deletes a connection
Description: Deletes a connection
Security: dlcon:connection:write
Response Code: 200
Description: Deletion OK
get /connections/{connectionId}
Returns a connection
Description: Returns the configuration details of this connection. Secrets such as passwords and access keys are not included in the response.
Security: dlcon:connection:read
Response Code: 200
Description: Configuration details of this connection
Schema:
{
"properties": {
"configuration": {
"description": "Configuration special to the connector type.\n\nThe exact content of this object vary based on the needs of the backend API the connector implementaton is designed to connect to.\nTypically includes network coordinates (eg. URL, hostname, port number, proxy configuraion) and credentials (eg. service account\nkeys or username / password) for accessing the underlying resource. Nothing outside this connector implementation needs to\nunderstand the contents of this object.\n\nImplementors should be careful not to include secrets when sending out a connector configuration (secrets should be treated as\nwrite-only properties).",
"type": "object"
},
"createdAt": {
"description": "The RFC3339 date and time when this connection was created.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"createdBy": {
"description": "The user that created the connection.",
"type": "string"
},
"id": {
"description": "Unique connection identifier",
"example": "615e48e5-2cf9-4968-95cd-2a61013b563b",
"format": "uuid",
"type": "string"
},
"type": {
"description": "The type of this connection. Must match the id of a ConnectionType supported by this connector service.",
"example": "azure-blobstore",
"type": "string"
},
"updatedAt": {
"description": "The RFC3339 date and time when the connection was last updated.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"updatedBy": {
"description": "The user that last updated the connection.",
"type": "string"
},
"version": {
"description": "Read-only property that changes every time this connection object is modified. Also returned in the eTag header when GETting the\nconnection. This value is required in the If-Match header of PATCH requests to support optimistic locking.",
"type": "string"
}
},
"required": [
"id",
"configuration",
"type",
"createdAt",
"createdBy"
],
"type": "object"
}patch /connections/{connectionId}
Updates a connection
Description: Updates the configuration details of this connection. Modifies the existing configuration details of this connection by applying the given JSON Patch document to it. Request body must be a JSON Patch document describing changes to a Connection object. Response headers conform to RFC 5789.
Security: dlcon:connection:write
Response Code: 204
Description: Update OK
Schema:
{
"properties": {
"configuration": {
"description": "Configuration special to the connector type.\n\nThe exact content of this object vary based on the needs of the backend API the connector implementaton is designed to connect to.\nTypically includes network coordinates (eg. URL, hostname, port number, proxy configuraion) and credentials (eg. service account\nkeys or username / password) for accessing the underlying resource. Nothing outside this connector implementation needs to\nunderstand the contents of this object.\n\nImplementors should be careful not to include secrets when sending out a connector configuration (secrets should be treated as\nwrite-only properties).",
"type": "object"
},
"createdAt": {
"description": "The RFC3339 date and time when this connection was created.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"createdBy": {
"description": "The user that created the connection.",
"type": "string"
},
"id": {
"description": "Unique connection identifier",
"example": "615e48e5-2cf9-4968-95cd-2a61013b563b",
"format": "uuid",
"type": "string"
},
"type": {
"description": "The type of this connection. Must match the id of a ConnectionType supported by this connector service.",
"example": "azure-blobstore",
"type": "string"
},
"updatedAt": {
"description": "The RFC3339 date and time when the connection was last updated.",
"example": "2021-01-01T11:12:13.000Z",
"format": "date-time",
"type": "string"
},
"updatedBy": {
"description": "The user that last updated the connection.",
"type": "string"
},
"version": {
"description": "Read-only property that changes every time this connection object is modified. Also returned in the eTag header when GETting the\nconnection. This value is required in the If-Match header of PATCH requests to support optimistic locking.",
"type": "string"
}
},
"required": [
"id",
"configuration",
"type",
"createdAt",
"createdBy"
],
"type": "object"
}get /connections/{connectionId}/blobs
Lists the blobs within this connection
Description: Lists the blobs within this connection. If this connection only supports tables, this endpoint returns an empty BlobList.
Security: dlcon:blob:info
Response Code: 200
Description: List of blobs within this connection
Schema:
{
"properties": {
"blobList": {
"items": {
"$ref": "#/components/schemas/Blob"
},
"type": "array"
},
"pagination": {
"$ref": "#/components/schemas/Pagination"
}
},
"required": [
"blobList"
],
"type": "object"
}post /connections/{connectionId}/blobs/access/https
Returns an access URL to fetch blob data
Description: Returns an Access URL describing how to fetch the blob data within this connection.
Security: dlcon:blob:data
Response Code: 200
Description: An access URL to fetch blob data
Schema:
{
"properties": {
"accessUrls": {
"items": {
"$ref": "/components/schemas/AccessURL"
},
"type": "array"
}
},
"type": "object"
}Response Code: 404
Description: The requested blob was not found within the requested connection.
get /connections/{connectionId}/health
Describes the health of the given connection
Description: Returns this connection’s Connected/Intermittent/Disconnected status based on the outcome of the last five calls to the backend API
Security: dlcon:connection:read
Response Code: 200
Description: Object with overall status, and a list of objects with details on the last five calls
Schema:
{
"properties": {
"recentCallList": {
"description": "List of the last five API calls made for a connection",
"items": {
"$ref": "#/components/schemas/RecentCall"
},
"type": "array"
},
"status": {
"description": "Overall health for the given connectionId - One of UP, INTERMITTENT or DOWN",
"type": "string"
}
},
"required": [
"status",
"recentCallList"
],
"type": "object"
}Response Code: 404
Description: The requested connectionId does not exist
Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}get /connections/{connectionId}/logs
Returns the logs of the given connection
Description: Returns the activity logs of the given data source connection.
Security: dlcon:connection:read
Response Code: 200
Description: List of activity logs
Schema:
{
"description": "List of all the logs associated with a connection (default up to 100)",
"items": {
"$ref": "#/components/schemas/RecentCall"
},
"type": "array"
}Response Code: 400
Description: The query parameters are invalid
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}Response Code: 404
Description: No records for the given connectionId found
Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}post /connections/{connectionId}/search
Performs a query over the tables of this connection (optional operation)
Description: Optional operation that accepts a SearchRequest and returns a TableData
Response Code: 200
Description: Query results returned as TableData
Schema:
{
"description": "A paginated collection of tabular data",
"properties": {
"data": {
"description": "Page of JSON values, each adhering to the schema given in the `dataModel` property",
"items": {
"type": "object"
},
"type": "array"
},
"dataModel": {
"$ref": "http://json-schema.org/draft-07/schema#"
},
"errors": {
"$ref": "#/components/schemas/ErrorList"
},
"pagination": {
"$ref": "#/components/schemas/Pagination"
}
},
"required": [
"dataModel",
"data"
],
"type": "object"
}Response Code: 400
Description: The query is invalid
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}Response Code: 404
Description: This connection does not implement the search operation
Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}get /connections/{connectionId}/table/{tableName}/data
Fetches data from a table
Description: Optional operation that returns data from a Table
Security: dlcon:table:data
Response Code: 200
Description: Data from the Table
Schema:
{
"description": "A paginated collection of tabular data",
"properties": {
"data": {
"description": "Page of JSON values, each adhering to the schema given in the `dataModel` property",
"items": {
"type": "object"
},
"type": "array"
},
"dataModel": {
"$ref": "http://json-schema.org/draft-07/schema#"
},
"errors": {
"$ref": "#/components/schemas/ErrorList"
},
"pagination": {
"$ref": "#/components/schemas/Pagination"
}
},
"required": [
"dataModel",
"data"
],
"type": "object"
}Response Code: 404
Description: The table does not exist within the requested connection
Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}get /connections/{connectionId}/table/{tableName}/info
Returns the schema of a table
Description: Returns the schema of the table identified by tableName within the requested connection
Security: dlcon:table:info
Response Code: 200
Description: A Table
Schema:
{
"description": "Describes a table of data.",
"properties": {
"dataModel": {
"$ref": "http://json-schema.org/draft-07/schema#"
},
"description": {
"description": "Optional description of the Table",
"type": "string"
},
"errors": {
"$ref": "#/components/schemas/ErrorList"
},
"flat_unique_name": {
"description": "dlcon extension: a one-part identifier (no dots) for this table that's unique within the connection this table belongs to.\nSome databases (like PostgreSQL) use three-part names but the first part is always the same within a given connection. Connectors\nfor such systems can use shorter flattened names that leave out the redundant prefix.\n\nExamples:\n\n**Simple table name**\n\nTable name from a database that doesn't have multiple schemas or tables\n\n```\nmy_table\n```\n\n**Two part table name**\n\nTable name from a database that uses two part names like schema.table or database.table\n\n```\npublic_my_table\n```\n\n**Three part table name**\n\nTable name from a database that uses three part names like catalog.schema.table or database.schema.table\n\n```\nmy_db_my_schema_my_table\n```\n",
"type": "string"
},
"name": {
"description": "Fully qualified table name.\n\nExamples:\n\n**Simple table name**\n\nTable name from a database that doesn't have multiple schemas or tables\n\n```\nmy_table\n```\n\n**Two part table name**\n\nTable name from a database that uses two part names like schema.table or database.table\n\n```\npublic.my_table\n```\n\n**Three part table name**\n\nTable name from a database that uses three part names like catalog.schema.table or database.schema.table\n\n```\nmy_db.my_schema.my_table\n```\n",
"type": "string"
}
},
"required": [
"name",
"dataModel"
],
"type": "object"
}Response Code: 404
Description: The requested table does not exist within the requested connection
Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}get /connections/{connectionId}/tables
Lists tables
Description: Lists the tables within the requested connection. If this connection does not contain any tables, returns an empty TablesList.
Security: dlcon:table:info
Response Code: 200
Description: List of tables within this connection.
Schema:
{
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
},
"pagination": {
"$ref": "#/components/schemas/Pagination"
},
"tables": {
"items": {
"$ref": "#/components/schemas/Table"
},
"type": "array"
}
},
"required": [
"tables"
],
"type": "object"
}Response Code: 500
Description: An unexpected error occurred
Schema:
{
"description": "The response body when no part of the request can be fulfilled",
"properties": {
"errors": {
"$ref": "#/components/schemas/ErrorList"
}
},
"type": "object"
}get /service-info
Provides information about this connector service
Description: Implementation of the GA4GH service-info API which provides information about this connector implementation
Response Code: 200
Description: Show information about this Connector service.
Use "type": { "group": "com.dnastack", "artifact": "datalake", "version": "1.0.0" } when implementing this specification directly.
Schema:
{
"$ref": "https://raw.githubusercontent.com/ga4gh-discovery/ga4gh-service-info/v1.0.0/service-info.yaml#/components/schemas/Service"
}