From 6936d93ecaf379ed80f0307233b5185466510dde Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 00:41:42 +0100 Subject: [PATCH 1/4] include custom fields in JQL query results convert list values to concatenated strings support definition of fieldMapping property in query which allows to rename output columns or pick specific members from dict values --- query_runner/jql.py | 51 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/query_runner/jql.py b/query_runner/jql.py index 50f110adfd..533f7ac562 100644 --- a/query_runner/jql.py +++ b/query_runner/jql.py @@ -26,15 +26,33 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue): +def parse_issue(issue, fieldMapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): - if k.startswith('customfield_'): - continue - if isinstance(v, dict): + # if field mapping is defined optionally change output key and parsing rules for value + if k in fieldMapping: + mapping = fieldMapping[k] + output_key = k + if 'name' in mapping: + output_key = mapping['name'] + put_value(result, output_key, v, mapping) + + else: + put_value(result, k, v, {}) + + return result + + +def put_value(result, k, v, mapping): + if isinstance(v, dict): + if 'member' in mapping: + result[k] = v[mapping['member']] + + else: + # these special mapping rules are kept for backwards compatibility if 'key' in v: result['{}_key'.format(k)] = v['key'] if 'name' in v: @@ -45,19 +63,27 @@ def parse_issue(issue): if 'watchCount' in v: result[k] = v['watchCount'] - # elif isinstance(v, list): - # pass - else: - result[k] = v + + elif isinstance(v, list): + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if 'member' in mapping: + listValues.append(listItem[mapping['member']]) + else: + listValues.append(listItem) - return result + result[k] = ','.join(listValues) + + else: + result[k] = v -def parse_issues(data): +def parse_issues(data, fieldMapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue)) + results.add_row(parse_issue(issue, fieldMapping)) return results @@ -109,6 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') + fieldMapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -127,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data) + results = parse_issues(data, fieldMapping) return results.to_json(), None except KeyboardInterrupt: From 15fda42f9917fddfe4a5621630df13fce9b66878 Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 12:15:30 +0100 Subject: [PATCH 2/4] should set nothing instead of empty string as value when no valid list items are found --- query_runner/jql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query_runner/jql.py b/query_runner/jql.py index 533f7ac562..dea1426c69 100644 --- a/query_runner/jql.py +++ b/query_runner/jql.py @@ -72,8 +72,8 @@ def put_value(result, k, v, mapping): listValues.append(listItem[mapping['member']]) else: listValues.append(listItem) - - result[k] = ','.join(listValues) + if len(listValues) > 0: + result[k] = ','.join(listValues) else: result[k] = v From 2a1e54e01a74aa5719035a30f46d4fc586f6831f Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 12:27:50 +0100 Subject: [PATCH 3/4] apply snake_case naming conventions --- query_runner/jql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/query_runner/jql.py b/query_runner/jql.py index dea1426c69..be17a306ae 100644 --- a/query_runner/jql.py +++ b/query_runner/jql.py @@ -26,15 +26,15 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue, fieldMapping): +def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): # if field mapping is defined optionally change output key and parsing rules for value - if k in fieldMapping: - mapping = fieldMapping[k] + if k in field_mapping: + mapping = field_mapping[k] output_key = k if 'name' in mapping: output_key = mapping['name'] @@ -79,11 +79,11 @@ def put_value(result, k, v, mapping): result[k] = v -def parse_issues(data, fieldMapping): +def parse_issues(data, field_mapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue, fieldMapping)) + results.add_row(parse_issue(issue, field_mapping)) return results @@ -135,7 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - fieldMapping = query.pop('fieldMapping', {}) + field_mapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -154,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data, fieldMapping) + results = parse_issues(data, field_mapping) return results.to_json(), None except KeyboardInterrupt: From c37308cec1470e9e7a5a025e96b5c4a6048e4607 Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 14:37:18 +0100 Subject: [PATCH 4/4] adapt to new field mapping syntax and add unit tests --- query_runner/jql.py | 127 +++++++++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/query_runner/jql.py b/query_runner/jql.py index be17a306ae..2b37339c48 100644 --- a/query_runner/jql.py +++ b/query_runner/jql.py @@ -1,5 +1,6 @@ import json import requests +import re from collections import OrderedDict @@ -30,53 +31,55 @@ def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] - for k, v in issue['fields'].iteritems(): + for k, v in issue['fields'].iteritems():# + output_name = field_mapping.get_output_field_name(k) + member_names = field_mapping.get_dict_members(k) - # if field mapping is defined optionally change output key and parsing rules for value - if k in field_mapping: - mapping = field_mapping[k] - output_key = k - if 'name' in mapping: - output_key = mapping['name'] - put_value(result, output_key, v, mapping) - - else: - put_value(result, k, v, {}) - - return result + if isinstance(v, dict): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + if member_name in v: + result[field_mapping.get_dict_output_field_name(k,member_name)] = v[member_name] + else: + # these special mapping rules are kept for backwards compatibility + if 'key' in v: + result['{}_key'.format(output_name)] = v['key'] + if 'name' in v: + result['{}_name'.format(output_name)] = v['name'] + + if k in v: + result[output_name] = v[k] + + if 'watchCount' in v: + result[output_name] = v['watchCount'] + + elif isinstance(v, list): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if member_name in listItem: + listValues.append(listItem[member_name]) + if len(listValues) > 0: + result[field_mapping.get_dict_output_field_name(k,member_name)] = ','.join(listValues) -def put_value(result, k, v, mapping): - if isinstance(v, dict): - if 'member' in mapping: - result[k] = v[mapping['member']] + else: + # otherwise support list values only for non-dict items + listValues = [] + for listItem in v: + if not isinstance(listItem, dict): + listValues.append(listItem) + if len(listValues) > 0: + result[output_name] = ','.join(listValues) else: - # these special mapping rules are kept for backwards compatibility - if 'key' in v: - result['{}_key'.format(k)] = v['key'] - if 'name' in v: - result['{}_name'.format(k)] = v['name'] - - if k in v: - result[k] = v[k] - - if 'watchCount' in v: - result[k] = v['watchCount'] - - elif isinstance(v, list): - listValues = [] - for listItem in v: - if isinstance(listItem, dict): - if 'member' in mapping: - listValues.append(listItem[mapping['member']]) - else: - listValues.append(listItem) - if len(listValues) > 0: - result[k] = ','.join(listValues) + result[output_name] = v - else: - result[k] = v + return result def parse_issues(data, field_mapping): @@ -94,6 +97,46 @@ def parse_count(data): return results +class FieldMapping: + + def __init__(cls, query_field_mapping): + cls.mapping = [] + for k, v in query_field_mapping.iteritems(): + field_name = k + member_name = None + + # check for member name contained in field name + member_parser = re.search('(\w+)\.(\w+)', k) + if (member_parser): + field_name = member_parser.group(1) + member_name = member_parser.group(2) + + cls.mapping.append({ + 'field_name': field_name, + 'member_name': member_name, + 'output_field_name': v + }) + + def get_output_field_name(cls,field_name): + for item in cls.mapping: + if item['field_name'] == field_name and not item['member_name']: + return item['output_field_name'] + return field_name + + def get_dict_members(cls,field_name): + member_names = [] + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name']: + member_names.append(item['member_name']) + return member_names + + def get_dict_output_field_name(cls,field_name, member_name): + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name'] == member_name: + return item['output_field_name'] + return None + + class JiraJQL(BaseQueryRunner): noop_query = '{"queryType": "count"}' @@ -135,7 +178,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - field_mapping = query.pop('fieldMapping', {}) + field_mapping = FieldMapping(query.pop('fieldMapping', {})) if query_type == 'count': query['maxResults'] = 1