Merge pull request #1138 from smartcanvas/feature/bigquery_udf

Add: ability to register user defined function (UDF) resources for BigQuery DataSource/Query
This commit is contained in:
Arik Fraimovich 2016-07-15 19:17:45 +03:00 committed by GitHub
commit 600afa5c82
2 changed files with 11 additions and 0 deletions

View File

@ -49,6 +49,8 @@ Google BigQuery
- Project ID (mandatory)
- JSON key file, generated when creating a service account (see `instructions <https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount>`__).
- UDF Source URIs (see `more details <https://cloud.google.com/bigquery/user-defined-functions#api>`__ )
- References to JavaScript source files stored in Google Cloud Storage (comma separated)
- **Additional requirements**:

View File

@ -99,6 +99,10 @@ class BigQuery(BaseQueryRunner):
'totalMBytesProcessedLimit': {
"type": "number",
'title': 'Total MByte Processed Limit'
},
'userDefinedFunctionResourceUri': {
"type": "string",
'title': 'UDF Source URIs (i.e. gs://bucket/date_utils.js, gs://bucket/string_utils.js )'
}
},
'required': ['jsonKeyFile', 'projectId'],
@ -142,6 +146,11 @@ class BigQuery(BaseQueryRunner):
}
}
if "userDefinedFunctionResourceUri" in self.configuration:
resource_uris = self.configuration["userDefinedFunctionResourceUri"].split(',')
job_data["configuration"]["query"]["userDefinedFunctionResources"] = map(
lambda resource_uri: {"resourceUri": resource_uri}, resource_uris)
insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
current_row = 0
query_reply = _get_query_results(jobs, project_id=project_id,