Usage Examples

Setup the MatillionClient¶

In [1]:

from pymatillion.matillion import MatillionClient

base_url = '<host url>'
username = '<username>'
password = '<password>'
project_group_name = '<project group name>'
project_name = '<project name>'

from pymatillion.matillion import MatillionClient base_url = '' username = '' password = '' project_group_name = '' project_name = ''

In [2]:

client = MatillionClient(
    base_url, 
    username, 
    password, 
    project_group_name, 
    project_name
)

client = MatillionClient( base_url, username, password, project_group_name, project_name )

List Project Groups¶

In [5]:

client.list_project_groups()

client.list_project_groups()

Out[5]:

['Sample_Project_Group']

List Projects¶

In [6]:

client.list_projects()

client.list_projects()

Out[6]:

['Sample_Project']

List Jobs¶

In [7]:

client.list_jobs()

client.list_jobs()

Out[7]:

['airports',
 'airports_orchestration_job',
 'examples README',
 'example datamodel ODS to 3NF',
 'example datamodel 3NF setup',
 'example datamodel load raw iot event',
 'example datamodel views ods to data vault',
 'example datamodel orchestrate data vault build',
 'example datamodel raw to staging',
 'example datamodel star schema ddl',
 'example datamodel run 3nf to star',
 'example datamodel xform 3nf to star part 1',
 'example datamodel xform 3nf to star part 2',
 'example datamodel xform 3nf to star part 3',
 'example datamodel run dv to star',
 'example datamodel xform dv to star part 1',
 'example datamodel xform dv to star part 2',
 'example datamodel xform dv to star part 3',
 'example datamodel setup ODS',
 'example datamodel staging to ODS',
 'example datamodel star to aggregate',
 'example PB2002 extract and load',
 'example PB2002 transformation',
 'example contingency table setup',
 'example surrogate key pattern setup',
 'example contingency table transform',
 'example surrogate key pattern transform',
 'example LAD EAF orchestration',
 'example LAD EAF staging',
 'example LAD EAF transform part 1',
 'example LAD EAF transform part 2',
 'example scd ddl',
 'example scd extract and load',
 'example scd ddl virtualization',
 'example scd type 1 update',
 'example scd type 2 update part 1',
 'example scd type 2 update part 2',
 'example scd update virtualized',
 'example scd viewer',
 'example densification extract load',
 'example densification transformation']

Run a Matillion Orchestration Job¶

• Matillion has two types of jobs, Transformation and Orchestration.
• The run_job() method can trigger Orchestration jobs only.
• You can wrap your Transformation jobs inside an Orchestration job and invoke them.

In [3]:

response = client.run_job(job_name='airports_orchestration_job')

response = client.run_job(job_name='airports_orchestration_job')

Get Task Details¶

• Once a job has been triggered, status and details of the job can be retreived by passing its id
• The run_job() endpoint, if successful, returns the id of the job execution.

In [4]:

client.get_task_details(task_id=response['id'])

client.get_task_details(task_id=response['id'])

Out[4]:

{'id': 887,
 'type': 'API_ORCHESTRATION',
 'customerID': 99,
 'groupName': 'Sample_Project_Group',
 'projectID': 129,
 'projectName': 'Sample_Project',
 'versionID': 130,
 'versionName': 'default',
 'jobID': 860,
 'jobName': 'airports_orchestration_job',
 'environmentID': 133,
 'environmentName': 'Snowflake',
 'state': 'RUNNING',
 'enqueuedTime': 1687994124261,
 'startTime': 1687994124298,
 'endTime': 0,
 'message': None,
 'originatorID': 's_7a31a297-d912-4e59-b1f5-ecd53d62f0a5',
 'rowCount': 0,
 'tasks': [{'taskID': 1,
   'parentID': -1,
   'type': 'VALIDATE_ORCHESTRATION',
   'jobID': 860,
   'jobName': 'airports_orchestration_job',
   'jobRevision': 4,
   'jobTimestamp': 1687993965936,
   'componentID': 861,
   'componentName': 'Start 0',
   'state': 'SUCCESS',
   'rowCount': -1,
   'startTime': 1687994124298,
   'endTime': 1687994124301,
   'message': '',
   'taskBatchID': 887},
  {'taskID': 2,
   'parentID': -1,
   'type': 'VALIDATE_ORCHESTRATION',
   'jobID': 860,
   'jobName': 'airports_orchestration_job',
   'jobRevision': 4,
   'jobTimestamp': 1687993965936,
   'componentID': 864,
   'componentName': 'airports 0',
   'state': 'RUNNING',
   'rowCount': -1,
   'startTime': 1687994124302,
   'endTime': -1,
   'message': '',
   'taskBatchID': 887},
  {'taskID': 4,
   'parentID': 2,
   'type': 'CREATE_VIEW',
   'jobID': 134,
   'jobName': 'airports',
   'jobRevision': 2,
   'jobTimestamp': 1687993230809,
   'componentID': 140,
   'componentName': 'Airport Data',
   'state': 'RUNNING',
   'rowCount': -1,
   'startTime': 1687994124312,
   'endTime': -1,
   'message': '',
   'taskBatchID': 887},
  {'taskID': 3,
   'parentID': 2,
   'type': 'CREATE_VIEW',
   'jobID': 134,
   'jobName': 'airports',
   'jobRevision': 2,
   'jobTimestamp': 1687993230809,
   'componentID': 141,
   'componentName': 'US States',
   'state': 'RUNNING',
   'rowCount': -1,
   'startTime': 1687994124312,
   'endTime': -1,
   'message': '',
   'taskBatchID': 887}],
 'hasHistoricJobs': True,
 'jobNames': ['airports_orchestration_job', 'airports']}