API Client represents a low level interface for Model API. All API operation can be performed using this.
abeja.models.
APIClient
(credential: typing.Union[dict, NoneType] = None, timeout: typing.Union[int, NoneType] = None, max_retry_count: typing.Union[int, NoneType] = None)¶A Low-Level client for Training API
from abeja.training import APIClient
api_client = APIClient()
archive_training_job
(organization_id: str, job_definition_name: str, training_job_id: str) → dict¶Archive a training job.
API reference: POST /organizations/<organization_id>/training/definitions/{job_definition_name}/jobs/{training_job_id}/archive
organization_id = "1102940376065"
job_definition_name = "test"
training_job_id = "1234567890123"
response = api_client.archive_training_job(organization_id,
job_definition_name,
training_job_id)
archive_training_job_definition
(organization_id: str, job_definition_name: str) → dict¶archive a training job definition
API reference: POST /organizations/<organization_id>/training/definitions/{name}/archive
organization_id = "1102940376065"
job_definition_name = "test"
response = api_client.archive_training_job_definition(organization_id,
job_definition_name)
archive_training_job_definition_version
(organization_id: str, job_definition_name: str, version_id: int) → dict¶archive a training job definition version
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>/archive
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
response = api_client.archive_training_job_definition_version(organization_id, job_definition_name, version_id)
Response Syntax:
{
"message": "archived"
}
archive_training_model
(organization_id: str, job_definition_name: str, model_id: str) → dict¶archive a training model
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/models/<model_id>/archive
response = api_client.archive_training_model(
organization_id='1111111111111', job_definition_name='1111111111111', model_id='1111111111111')
{
"message": "{job_definition_name}:{model_id} archived"
}
create_training_job
(organization_id: str, job_definition_name: str, version_id: int, user_parameters: typing.Union[dict, NoneType] = None, datasets: typing.Union[dict, NoneType] = None, instance_type: typing.Union[str, NoneType] = None, environment: typing.Union[dict, NoneType] = None, description: typing.Union[str, NoneType] = None, export_log: typing.Union[bool, NoneType] = None) → dict¶create a training job
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>/jobs
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
user_parameters = {
'BATCH_SIZE': 50
}
datasets = {
"mnist": "1111111111111"
}
response = api_client.create_training_job(
organization_id, job_definition_name, version_id, user_parameters, datasets)
environment
instead.true
, include the log in the model.
This feature is only available with 19.04 or later images. (default: false
)Response Syntax:
{
"job_definition_id": "1443714239154",
"user_parameters": {},
"start_time": null,
"created_at": "2018-05-17T12:43:59.322367Z",
"job_definition_version": 1,
"completion_time": null,
"status": "Pending",
"instance_type": "cpu-1",
"modified_at": "2018-05-17T12:43:59.322673Z",
"training_job_id": "1443722127663",
"creator": {
"email": "test@abeja.asia",
"is_registered": true,
"created_at": "2017-05-26T01:38:46Z",
"id": "1128347408389",
"display_name": null,
"updated_at": "2018-01-04T03:02:12Z",
"role": "admin"
},
"description": null,
"statistics": null
}
create_training_job_definition
(organization_id: str, job_definition_name: str) → dict¶create a training job definition
API reference: POST /organizations/<organization_id>/training/definitions
organization_id = "1102940376065"
job_definition_name = "test"
response = api_client.create_training_job_definition(organization_id,
job_definition_name)
Response Syntax:
{
"job_definition_id": "1443334816413",
"versions": [],
"organization_id": "1200123565071",
"modified_at": "2018-05-17T02:13:35.726812Z",
"created_at": "2018-05-17T02:13:35.726691Z",
"version_count": 0,
"name": "test"
}
create_training_job_definition_version
(organization_id: str, job_definition_name: str, filepaths: typing.List[str], handler: str, image: typing.Union[str, NoneType] = None, environment: typing.Union[typing.Dict[str, typing.Any], NoneType] = None, description: typing.Union[str, NoneType] = None) → dict¶create a training job definition version.
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/versions
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
filepaths = ["./requirements.txt", "./train.py"]
handler = "train:handler"
image = "abeja-inc/all-gpu:19.04"
environment = {"key": "value"}
description = "description"
response = api_client.create_training_job_definition_version(
organization_id, job_definition_name, filepaths, handler,
image=image, environment=environment, description=description)
Response Syntax:
{
"job_definition_version": 1,
"user_parameters": {},
"environment": {},
"description": "description",
"datasets": {
"mnist": "1111111111111"
},
"modified_at": "2018-05-17T12:34:46.344076Z",
"job_definition_id": "1443714239154",
"handler": "train:handler",
"created_at": "2018-05-17T12:34:46.296488Z",
"image": "abeja-inc/all-gpu:19.04"
}
create_training_job_definition_version_native_api
(organization_id: str, job_definition_name: str, source_code: typing.IO[AnyStr], parameters: dict) → dict¶create a training job definition version.
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/versions
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
source_code = open("./train.zip", "rb")
handler = "train:handler"
image = "abeja-inc/all-gpu:19.04"
environment = {"key": "value"}
description = "description"
response = api_client.create_training_job_definition_version_native_api(
organization_id, job_definition_name, source_code,
parameters={"handler": handler, "image": image, "environment": environment, "description": description})
Response Syntax:
{
"job_definition_version": 1,
"user_parameters": {},
"environment": {},
"description": "description",
"datasets": {
"mnist": "1111111111111"
},
"modified_at": "2018-05-17T12:34:46.344076Z",
"job_definition_id": "1443714239154",
"handler": "train:handler",
"created_at": "2018-05-17T12:34:46.296488Z",
"image": "abeja-inc/all-gpu:19.04"
}
create_training_model
(organization_id: str, job_definition_name: str, model_data: typing.IO, parameters: typing.Union[dict, NoneType] = None) → dict¶create a training model.
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/models
organization_id = "1102940376065"
job_definition_name = 'test_job_definition'
model_data = '....'
parameters = {
"description": "description",
"user_parameters": {}
}
response = api_client.create_training_model(
organization_id, job_definition_name, model_data, parameters)
Response Syntax:
{
"training_model_id": "1111111111111",
"job_definition_id": "1111111111111",
"training_job_id": "1111111111111",
"user_parameters": {},
"description": "this is description of the model",
"archived": false,
"exec_env": "cloud",
"created_at": "2018-01-01T00:00:00.00000Z",
"modified_at": "2018-01-01T00:00:00.00000Z"
}
delete_training_job_definition
(organization_id: str, job_definition_name: str) → dict¶delete a training job definition.
API reference: DELETE /organizations/<organization_id>/training/definitions/<job_definition_name>
organization_id = "1102940376065"
job_definition_name = 'test'
response = api_client.delete_training_job_definition(organization_id, job_definition_name)
Response Syntax:
{
"message": "test deleted"
}
delete_training_job_definition_version
(organization_id: str, job_definition_name: str, version_id: int) → dict¶delete a training job definition version
API reference: DELETE /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
response = api_client.delete_training_job_definition_version(organization_id, job_definition_name, version_id)
Response Syntax:
{
"message": "deleted"
}
download_training_model
(organization_id: str, job_definition_name: str, model_id: str) → dict¶download a training model
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/models/<model_id>/download
response = api_client.download_training_model(
organization_id='1111111111111', job_definition_name='1111111111111', model_id='1111111111111')
{
"download_uri": "https://..."
}
get_training_job
(organization_id: str, job_definition_name: str, training_job_id: str) → dict¶get a training job
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/jobs/<training_job_id>
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
training_job_id = "1443722127663"
response = api_client.get_training_job(organization_id, job_definition_name, training_job_id)
Response Syntax:
{
"job_definition_id": "1443714239154",
"user_parameters": {},
"start_time": null,
"created_at": "2018-05-17T12:43:59.322367Z",
"job_definition_version": 1,
"completion_time": null,
"status": "Pending",
"modified_at": "2018-05-17T12:43:59.322673Z",
"training_job_id": "1443722127663",
"archived": false,
"creator": {
"email": "test@abeja.asia",
"is_registered": true,
"created_at": "2017-05-26T01:38:46Z",
"id": "1128347408389",
"display_name": null,
"updated_at": "2018-01-04T03:02:12Z",
"role": "admin"
},
"description": null,
"statistics": null
}
get_training_job_definition
(organization_id: str, job_definition_name: str, include_jobs: typing.Union[bool, NoneType] = None) → dict¶get a training job definition.
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>
organization_id = "1102940376065"
job_definition_name = 'test'
response = api_client.get_training_job_definition(organization_id, job_definition_name)
True
, also returns training jobs in response. By historical reason,Response Syntax:
{
"modified_at": "2018-05-17T02:13:35.726812Z",
"organization_id": "1200123565071",
"created_at": "2018-05-17T02:13:35.726691Z",
"job_definition_id": "1443334816413",
"name": "test",
"archived": false,
"versions": [],
"version_count": 0
}
get_training_job_definition_version
(organization_id: str, job_definition_name: str, version_id: int) → dict¶get a training job definition version
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
response = api_client.get_training_job_definition_version(organization_id, job_definition_name, version_id)
Response Syntax:
{
"job_definition_version": 1,
"user_parameters": {},
"datasets": {
"mnist": "1111111111111"
},
"modified_at": "2018-05-17T12:34:46.344076Z",
"job_definition_id": "1443714239154",
"handler": "train:handler",
"created_at": "2018-05-17T12:34:46.296488Z",
"image": "abeja-inc/all-gpu:19.04",
"archived": false
}
get_training_job_definition_versions
(organization_id: str, job_definition_name: str, filter_archived: typing.Union[bool, NoneType] = None) → dict¶get training job definition versions.
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/versions
organization_id = "1102940376065"
job_definition_name = 'test_job_definition'
response = api_client.get_training_job_definition_versions(organization_id, job_definition_name)
true
, include archived jobs, otherwise exclude archived jobs. (default: false
)Response Syntax:
{
"entries": [
{
"job_definition_version": 1,
"user_parameters": {},
"datasets": {
"mnist": "1111111111111"
},
"modified_at": "2018-05-17T12:34:46.344076Z",
"job_definition_id": "1443714239154",
"handler": "train:handler",
"created_at": "2018-05-17T12:34:46.296488Z",
"image": "abeja-inc/all-gpu:19.04",
"archived": false
}
]
}
get_training_job_definitions
(organization_id: str, filter_archived: typing.Union[bool, NoneType] = None, offset: typing.Union[int, NoneType] = None, limit: typing.Union[int, NoneType] = None) → dict¶get training job definitions
API reference: GET /organizations/<organization_id>/training/definitions
organization_id = "1102940376065"
response = api_client.get_training_job_definitions(organization_id)
true
, include archived jobs, otherwise exclude archived jobs. (default: false
)Response Syntax:
{
"entries": [
{
"version_count": 1,
"created_at": "2018-03-08T00:46:50.791787Z",
"organization_id": "1200123565071",
"versions": [
{
"job_definition_version": 1,
"user_parameters": {},
"handler": "train:handler",
"image": "abeja-inc/all-gpu:19.04",
"modified_at": "2018-03-08T00:48:12.207883Z",
"datasets": {
"train": "1376063797251"
},
"created_at": "2018-03-08T00:48:12.132471Z",
"job_definition_id": "1381349997580"
}
],
"name": "test",
"archived": false,
"modified_at": "2018-03-08T00:46:50.791946Z",
"job_definition_id": "1381349997580"
}
]
}
get_training_jobs
(organization_id: str, job_definition_name: str, limit: typing.Union[int, NoneType] = None, offset: typing.Union[int, NoneType] = None, filter_archived: typing.Union[bool, NoneType] = None) → dict¶get training jobs
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/jobs
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
response = api_client.get_training_jobs(organization_id, job_definition_name)
true
, include archived jobs, otherwise exclude archived jobs. (default: false
)Response Syntax:
{
"entries": [
{
"user_parameters": {},
"start_time": null,
"training_job_id": "1443722127663",
"created_at": "2018-05-17T12:43:59.322367Z",
"completion_time": null,
"id": "1443722127663",
"job_definition_version": 1,
"description": null,
"statistics": null,
"job_definition_id": "1443714239154",
"modified_at": "2018-05-17T12:43:59.322673Z",
"status": "Pending",
"archived": false,
"creator": {
"email": "test@abeja.asia",
"created_at": "2017-05-26T01:38:46Z",
"id": "1128347408389",
"role": "admin",
"display_name": null,
"updated_at": "2018-01-04T03:02:12Z",
"is_registered": true
}
}
],
"limit": 10,
"offset": 0,
"total": 1
}
get_training_model
(organization_id: str, job_definition_name: str, model_id: str) → dict¶get a training model
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/models/<model_id>
response = api_client.get_training_model(
organization_id='1111111111111', job_definition_name='1111111111111', model_id='1111111111111')
{
"training_model_id": "1111111111111",
"job_definition_id": "1111111111111",
"training_job_id": "1111111111111",
"user_parameters": {},
"description": "this is description of the model",
"archived": false,
"exec_env": "cloud",
"archived": false,
"created_at": "2018-01-01T00:00:00.00000Z",
"modified_at": "2018-01-01T00:00:00.00000Z"
}
get_training_models
(organization_id: str, job_definition_name: str, filter_archived: typing.Union[bool, NoneType] = None) → dict¶Get models entries
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/models
response = api_client.list_models(organization_id='1102940376065')
true
, include archived jobs, otherwise exclude archived jobs. (default: false
){
"entries": [
{
"training_model_id": "1111111111111",
"job_definition_id": "1111111111111",
"training_job_id": "1111111111111",
"user_parameters": {},
"description": "this is description of the model",
"archived": false,
"exec_env": "cloud",
"archived": false,
"created_at": "2018-01-01T00:00:00.00000Z",
"modified_at": "2018-01-01T00:00:00.00000Z"
}
]
}
get_training_result
(organization_id: str, job_definition_name: str, training_job_id: str) → dict¶get a training job result
API reference: GET /organizations/<organization_id>/training/definitions/<job_definition_name>/jobs/<training_job_id>/result
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
training_job_id = "1443722127663"
response = api_client.get_training_result(organization_id, job_definition_name,
training_job_id)
Response Syntax:
{
"artifacts": {
"complete": {
"uri": "dummy_url",
}
}
}
patch_training_job_definition_version
(organization_id: str, job_definition_name: str, version_id: int, description: str) → dict¶Update a training job definition version
API reference: PATCH /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
response = api_client.patch_training_job_definition_version(organization_id, job_definition_name, version_id, description='new version')
Response Syntax:
{
"job_definition_version": 1,
"user_parameters": {},
"datasets": {
"mnist": "1111111111111"
},
"modified_at": "2018-05-17T12:34:46.344076Z",
"job_definition_id": "1443714239154",
"handler": "train:handler",
"created_at": "2018-05-17T12:34:46.296488Z",
"image": "abeja-inc/all-gpu:19.04",
"archived": false
}
patch_training_model
(organization_id: str, job_definition_name: str, model_id: str, description: str) → dict¶patch a training model
API reference: PATCH /organizations/<organization_id>/training/definitions/<job_definition_name>/models/<model_id>
response = api_client.patch_training_model(
organization_id='1111111111111', job_definition_name='1111111111111',
model_id='1111111111111', description='new description')
{
"training_model_id": "1111111111111",
"job_definition_id": "1111111111111",
"training_job_id": "1111111111111",
"user_parameters": {},
"description": "this is description of the model",
"archived": false,
"exec_env": "cloud",
"created_at": "2018-01-01T00:00:00.00000Z",
"modified_at": "2018-01-01T00:00:00.00000Z"
}
stop_training_job
(organization_id: str, job_definition_name: str, training_job_id: str) → dict¶stop a training job
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/jobs/<training_job_id>/stop
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
training_job_id = "1443722127663"
response = api_client.stop_training_job(organization_id, job_definition_name, training_job_id)
Response Syntax:
{
"message": "test_job_definition:1443722127663 stopped"
}
unarchive_training_job
(organization_id: str, job_definition_name: str, training_job_id: str) → dict¶Archive a training job.
API reference: POST /organizations/<organization_id>/training/definitions/{job_definition_name}/jobs/{training_job_id}/unarchive
organization_id = "1102940376065"
job_definition_name = "test"
training_job_id = "1234567890123"
response = api_client.unarchive_training_job(organization_id,
job_definition_name,
training_job_id)
unarchive_training_job_definition
(organization_id: str, job_definition_name: str) → dict¶unarchive a training job definition
API reference: POST /organizations/<organization_id>/training/definitions/{name}/unarchive
organization_id = "1102940376065"
job_definition_name = "test"
response = api_client.unarchive_training_job_definition(organization_id,
job_definition_name)
unarchive_training_job_definition_version
(organization_id: str, job_definition_name: str, version_id: int) → dict¶unarchive a training job definition version
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/versions/<version_id>/unarchive
organization_id = "1102940376065"
job_definition_name = "test_job_definition"
version_id = 1
response = api_client.unarchive_training_job_definition_version(organization_id, job_definition_name, version_id)
Response Syntax:
{
"message": "unarchived"
}
unarchive_training_model
(organization_id: str, job_definition_name: str, model_id: str) → dict¶unarchive a training model
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/models/<model_id>/unarchive
response = api_client.unarchive_training_model(
organization_id='1111111111111', job_definition_name='1111111111111', model_id='1111111111111')
{
"message": "{job_definition_name}:{model_id} unarchived"
}
update_statistics
(organization_id: str, job_definition_name: str, training_job_id: str, statistics: dict) → dict¶update a training job statistics
API reference: POST /organizations/<organization_id>/training/definitions/<job_definition_name>/jobs/<training_job_id>/statistics
from abeja.training.statistics import Statistics
statistics = Statistics(progress_percentage=0.5, epoch=1, num_epochs=5, key1='value1')
statistics.add_stage(name=Statistics.STAGE_TRAIN, accuracy=0.9, loss=0.05)
statistics.add_stage(name=Statistics.STAGE_VALIDATION, accuracy=0.8, loss=0.1, key2=2)
response = api_client.update_statistics(statistics.get_statistics())
Response Syntax:
{
"statistics": {
"num_epochs": 5,
"epoch": 1,
"progress_percentage": 0.5,
"stages": {
"train": {
"accuracy": 0.9,
"loss": 0.05
},
"validation": {
"accuracy": 0.8,
"loss": 0.1,
"key2": 2
}
},
"key1": "value1"
}
}