1# SPDX-License-Identifier: GPL-2.0+
2#
3# Copyright 2020 Google LLC
4#
5"""Talks to the patchwork service to figure out what patches have been reviewed
6and commented on. Provides a way to display review tags and comments.
7Allows creation of a new branch based on the old but with the review tags
8collected from patchwork.
9"""
10
11import asyncio
12from collections import defaultdict
13import concurrent.futures
14from itertools import repeat
15
16import aiohttp
17import pygit2
18
19from u_boot_pylib import terminal
20from u_boot_pylib import tout
21from patman import patchstream
22from patman import patchwork
23
24
25def process_reviews(content, comment_data, base_rtags):
26    """Process and return review data
27
28    Args:
29        content (str): Content text of the patch itself - see pwork.get_patch()
30        comment_data (list of dict): Comments for the patch - see
31            pwork._get_patch_comments()
32        base_rtags (dict): base review tags (before any comments)
33            key: Response tag (e.g. 'Reviewed-by')
34            value: Set of people who gave that response, each a name/email
35                string
36
37    Return: tuple:
38        dict: new review tags (noticed since the base_rtags)
39            key: Response tag (e.g. 'Reviewed-by')
40            value: Set of people who gave that response, each a name/email
41                string
42        list of patchwork.Review: reviews received on the patch
43    """
44    pstrm = patchstream.PatchStream.process_text(content, True)
45    rtags = defaultdict(set)
46    for response, people in pstrm.commit.rtags.items():
47        rtags[response].update(people)
48
49    reviews = []
50    for comment in comment_data:
51        pstrm = patchstream.PatchStream.process_text(comment['content'], True)
52        if pstrm.snippets:
53            submitter = comment['submitter']
54            person = f"{submitter['name']} <{submitter['email']}>"
55            reviews.append(patchwork.Review(person, pstrm.snippets))
56        for response, people in pstrm.commit.rtags.items():
57            rtags[response].update(people)
58
59    # Find the tags that are not in the commit
60    new_rtags = defaultdict(set)
61    for tag, people in rtags.items():
62        for who in people:
63            is_new = (tag not in base_rtags or
64                      who not in base_rtags[tag])
65            if is_new:
66                new_rtags[tag].add(who)
67    return new_rtags, reviews
68
69
70def compare_with_series(series, patches):
71    """Compare a list of patches with a series it came from
72
73    This prints any problems as warnings
74
75    Args:
76        series (Series): Series to compare against
77        patches (list of Patch): list of Patch objects to compare with
78
79    Returns:
80        tuple
81            dict:
82                key: Commit number (0...n-1)
83                value: Patch object for that commit
84            dict:
85                key: Patch number  (0...n-1)
86                value: Commit object for that patch
87    """
88    # Check the names match
89    warnings = []
90    patch_for_commit = {}
91    all_patches = set(patches)
92    for seq, cmt in enumerate(series.commits):
93        pmatch = [p for p in all_patches if p.subject == cmt.subject]
94        if len(pmatch) == 1:
95            patch_for_commit[seq] = pmatch[0]
96            all_patches.remove(pmatch[0])
97        elif len(pmatch) > 1:
98            warnings.append("Multiple patches match commit %d ('%s'):\n   %s" %
99                            (seq + 1, cmt.subject,
100                             '\n   '.join([p.subject for p in pmatch])))
101        else:
102            warnings.append("Cannot find patch for commit %d ('%s')" %
103                            (seq + 1, cmt.subject))
104
105    # Check the names match
106    commit_for_patch = {}
107    all_commits = set(series.commits)
108    for seq, patch in enumerate(patches):
109        cmatch = [c for c in all_commits if c.subject == patch.subject]
110        if len(cmatch) == 1:
111            commit_for_patch[seq] = cmatch[0]
112            all_commits.remove(cmatch[0])
113        elif len(cmatch) > 1:
114            warnings.append("Multiple commits match patch %d ('%s'):\n   %s" %
115                            (seq + 1, patch.subject,
116                             '\n   '.join([c.subject for c in cmatch])))
117        else:
118            warnings.append("Cannot find commit for patch %d ('%s')" %
119                            (seq + 1, patch.subject))
120
121    return patch_for_commit, commit_for_patch, warnings
122
123
124def show_responses(col, rtags, indent, is_new):
125    """Show rtags collected
126
127    Args:
128        col (terminal.Colour): Colour object to use
129        rtags (dict): review tags to show
130            key: Response tag (e.g. 'Reviewed-by')
131            value: Set of people who gave that response, each a name/email string
132        indent (str): Indentation string to write before each line
133        is_new (bool): True if this output should be highlighted
134
135    Returns:
136        int: Number of review tags displayed
137    """
138    count = 0
139    for tag in sorted(rtags.keys()):
140        people = rtags[tag]
141        for who in sorted(people):
142            terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
143                           newline=False, colour=col.GREEN, bright=is_new,
144                           col=col)
145            terminal.tprint(who, colour=col.WHITE, bright=is_new, col=col)
146            count += 1
147    return count
148
149def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
150                  repo=None):
151    """Create a new branch with review tags added
152
153    Args:
154        series (Series): Series object for the existing branch
155        new_rtag_list (list): List of review tags to add, one for each commit,
156                each a dict:
157            key: Response tag (e.g. 'Reviewed-by')
158            value: Set of people who gave that response, each a name/email
159                string
160        branch (str): Existing branch to update
161        dest_branch (str): Name of new branch to create
162        overwrite (bool): True to force overwriting dest_branch if it exists
163        repo (pygit2.Repository): Repo to use (use None unless testing)
164
165    Returns:
166        int: Total number of review tags added across all commits
167
168    Raises:
169        ValueError: if the destination branch name is the same as the original
170            branch, or it already exists and @overwrite is False
171    """
172    if branch == dest_branch:
173        raise ValueError(
174            'Destination branch must not be the same as the original branch')
175    if not repo:
176        repo = pygit2.Repository('.')
177    count = len(series.commits)
178    new_br = repo.branches.get(dest_branch)
179    if new_br:
180        if not overwrite:
181            raise ValueError("Branch '%s' already exists (-f to overwrite)" %
182                             dest_branch)
183        new_br.delete()
184    if not branch:
185        branch = 'HEAD'
186    target = repo.revparse_single('%s~%d' % (branch, count))
187    repo.branches.local.create(dest_branch, target)
188
189    num_added = 0
190    for seq in range(count):
191        parent = repo.branches.get(dest_branch)
192        cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
193
194        repo.merge_base(cherry.oid, parent.target)
195        base_tree = cherry.parents[0].tree
196
197        index = repo.merge_trees(base_tree, parent, cherry)
198        tree_id = index.write_tree(repo)
199
200        lines = []
201        if new_rtag_list[seq]:
202            for tag, people in new_rtag_list[seq].items():
203                for who in people:
204                    lines.append('%s: %s' % (tag, who))
205                    num_added += 1
206        message = patchstream.insert_tags(cherry.message.rstrip(),
207                                          sorted(lines))
208
209        repo.create_commit(
210            parent.name, cherry.author, cherry.committer, message, tree_id,
211            [parent.target])
212    return num_added
213
214
215def check_patch_count(num_commits, num_patches):
216    """Check the number of commits and patches agree
217
218    Args:
219        num_commits (int): Number of commits
220        num_patches (int): Number of patches
221    """
222    if num_patches != num_commits:
223        tout.warning(f'Warning: Patchwork reports {num_patches} patches, '
224                     f'series has {num_commits}')
225
226
227def do_show_status(series, cover, patches, show_comments, show_cover_comments,
228                   col, warnings_on_stderr=True):
229    """Check the status of a series on Patchwork
230
231    This finds review tags and comments for a series in Patchwork, displaying
232    them to show what is new compared to the local series.
233
234    Args:
235        series (Series): Series object for the existing branch
236        cover (COVER): Cover letter info, or None if none
237        patches (list of Patch): Patches sorted by sequence number
238        show_comments (bool): True to show the comments on each patch
239        show_cover_comments (bool): True to show the comments on the
240            letter
241        col (terminal.Colour): Colour object
242
243    Return: tuple:
244        int: Number of new review tags to add
245        list: List of review tags to add, one item for each commit, each a
246                dict:
247            key: Response tag (e.g. 'Reviewed-by')
248            value: Set of people who gave that response, each a name/email
249                string
250    """
251    compare = []
252    for pw_patch in patches:
253        patch = patchwork.Patch(pw_patch.id)
254        patch.parse_subject(pw_patch.series_data['name'])
255        compare.append(patch)
256
257    count = len(series.commits)
258    new_rtag_list = [None] * count
259    review_list = [None] * count
260
261    with terminal.pager():
262        patch_for_commit, _, warnings = compare_with_series(series, compare)
263        for warn in warnings:
264            tout.do_output(tout.WARNING if warnings_on_stderr else tout.INFO,
265                           warn)
266
267        for seq, pw_patch in enumerate(patches):
268            compare[seq].patch = pw_patch
269
270        for i in range(count):
271            pat = patch_for_commit.get(i)
272            if pat:
273                patch_data = pat.patch.data
274                comment_data = pat.patch.comments
275                new_rtag_list[i], review_list[i] = process_reviews(
276                    patch_data['content'], comment_data,
277                    series.commits[i].rtags)
278        num_to_add = _do_show_status(
279            series, cover, patch_for_commit, show_comments,
280            show_cover_comments, new_rtag_list, review_list, col)
281
282    return num_to_add, new_rtag_list
283
284
285def _do_show_status(series, cover, patch_for_commit, show_comments,
286                    show_cover_comments, new_rtag_list, review_list, col):
287    if cover and show_cover_comments:
288        terminal.tprint(f'Cov {cover.name}', colour=col.BLACK, col=col,
289                        bright=False, back=col.YELLOW)
290        for seq, comment in enumerate(cover.comments):
291            submitter = comment['submitter']
292            person = '%s <%s>' % (submitter['name'], submitter['email'])
293            terminal.tprint(f"From: {person}: {comment['date']}",
294                            colour=col.RED, col=col)
295            print(comment['content'])
296            print()
297
298    num_to_add = 0
299    for seq, cmt in enumerate(series.commits):
300        patch = patch_for_commit.get(seq)
301        if not patch:
302            continue
303        terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]),
304                       colour=col.YELLOW, col=col)
305        cmt = series.commits[seq]
306        base_rtags = cmt.rtags
307        new_rtags = new_rtag_list[seq]
308
309        indent = ' ' * 2
310        show_responses(col, base_rtags, indent, False)
311        num_to_add += show_responses(col, new_rtags, indent, True)
312        if show_comments:
313            for review in review_list[seq]:
314                terminal.tprint('Review: %s' % review.meta, colour=col.RED,
315                                col=col)
316                for snippet in review.snippets:
317                    for line in snippet:
318                        quoted = line.startswith('>')
319                        terminal.tprint(
320                            f'    {line}',
321                            colour=col.MAGENTA if quoted else None, col=col)
322                    terminal.tprint()
323    return num_to_add
324
325
326def show_status(series, branch, dest_branch, force, cover, patches,
327                show_comments, show_cover_comments, test_repo=None):
328    """Check the status of a series on Patchwork
329
330    This finds review tags and comments for a series in Patchwork, displaying
331    them to show what is new compared to the local series.
332
333    Args:
334        client (aiohttp.ClientSession): Session to use
335        series (Series): Series object for the existing branch
336        branch (str): Existing branch to update, or None
337        dest_branch (str): Name of new branch to create, or None
338        force (bool): True to force overwriting dest_branch if it exists
339        cover (COVER): Cover letter info, or None if none
340        patches (list of Patch): Patches sorted by sequence number
341        show_comments (bool): True to show the comments on each patch
342        show_cover_comments (bool): True to show the comments on the letter
343        test_repo (pygit2.Repository): Repo to use (use None unless testing)
344    """
345    col = terminal.Color()
346    check_patch_count(len(series.commits), len(patches))
347    num_to_add, new_rtag_list = do_show_status(
348        series, cover, patches, show_comments, show_cover_comments, col)
349
350    if not dest_branch and num_to_add:
351        msg = ' (use -d to write them to a new branch)'
352    else:
353        msg = ''
354    terminal.tprint(
355        f"{num_to_add} new response{'s' if num_to_add != 1 else ''} "
356        f'available in patchwork{msg}')
357
358    if dest_branch:
359        num_added = create_branch(series, new_rtag_list, branch,
360                                  dest_branch, force, test_repo)
361        terminal.tprint(
362            f"{num_added} response{'s' if num_added != 1 else ''} added "
363            f"from patchwork into new branch '{dest_branch}'")
364
365
366async def check_status(link, pwork, read_comments=False,
367                       read_cover_comments=False):
368    """Set up an HTTP session and get the required state
369
370    Args:
371        link (str): Patch series ID number
372        pwork (Patchwork): Patchwork object to use for reading
373        read_comments (bool): True to read comments and state for each patch
374
375        Return: tuple:
376            COVER object, or None if none or not read_cover_comments
377            list of PATCH objects
378    """
379    async with aiohttp.ClientSession() as client:
380        return await pwork.series_get_state(client, link, read_comments,
381                                             read_cover_comments)
382
383
384def check_and_show_status(series, link, branch, dest_branch, force,
385                          show_comments, show_cover_comments, pwork,
386                          test_repo=None):
387    """Read the series status from patchwork and show it to the user
388
389    Args:
390        series (Series): Series object for the existing branch
391        link (str): Patch series ID number
392        branch (str): Existing branch to update, or None
393        dest_branch (str): Name of new branch to create, or None
394        force (bool): True to force overwriting dest_branch if it exists
395        show_comments (bool): True to show the comments on each patch
396        show_cover_comments (bool): True to show the comments on the letter
397        pwork (Patchwork): Patchwork object to use for reading
398        test_repo (pygit2.Repository): Repo to use (use None unless testing)
399    """
400    loop = asyncio.get_event_loop()
401    cover, patches = loop.run_until_complete(check_status(
402        link, pwork, True, show_cover_comments))
403
404    show_status(series, branch, dest_branch, force, cover, patches,
405                show_comments, show_cover_comments, test_repo=test_repo)
406