1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2011 The Chromium OS Authors. 3# 4 5"""Handles parsing a stream of commits/emails from 'git log' or other source""" 6 7import collections 8import datetime 9import io 10import math 11import os 12import re 13import queue 14import shutil 15import tempfile 16 17from patman import commit 18from patman.series import Series 19from u_boot_pylib import command 20from u_boot_pylib import gitutil 21 22# Tags that we detect and remove 23RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' 24 r'|Reviewed-on:|Commit-\w*:') 25 26# Lines which are allowed after a TEST= line 27RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:') 28 29# Signoffs 30RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)') 31 32# Cover letter tag 33RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)') 34 35# Patch series tag 36RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)') 37 38# Change-Id will be used to generate the Message-Id and then be stripped 39RE_CHANGE_ID = re.compile('^Change-Id: *(.*)') 40 41# Commit series tag 42RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)') 43 44# Commit tags that we want to collect and keep 45RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)') 46 47# The start of a new commit in the git log 48RE_COMMIT = re.compile('^commit ([0-9a-f]*)$') 49 50# We detect these since checkpatch doesn't always do it 51RE_SPACE_BEFORE_TAB = re.compile(r'^[+].* \t') 52 53# Match indented lines for changes 54RE_LEADING_WHITESPACE = re.compile(r'^\s') 55 56# Detect a 'diff' line 57RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$') 58 59# Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch 60RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)') 61 62# Detect line with invalid TAG 63RE_INV_TAG = re.compile('^Serie-([a-z-]*): *(.*)') 64 65# States we can be in - can we use range() and still have comments? 66STATE_MSG_HEADER = 0 # Still in the message header 67STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 68STATE_PATCH_HEADER = 2 # In patch header (after the subject) 69STATE_DIFFS = 3 # In the diff part (past --- line) 70 71 72class PatchStream: 73 """Class for detecting/injecting tags in a patch or series of patches 74 75 We support processing the output of 'git log' to read out the tags we 76 are interested in. We can also process a patch file in order to remove 77 unwanted tags or inject additional ones. These correspond to the two 78 phases of processing. 79 80 Args: 81 keep_change_id (bool): Keep the Change-Id tag 82 insert_base_commit (bool): True to add the base commit to the end 83 """ 84 def __init__(self, series, is_log=False, keep_change_id=False, 85 insert_base_commit=False): 86 self.skip_blank = False # True to skip a single blank line 87 self.found_test = False # Found a TEST= line 88 self.lines_after_test = 0 # Number of lines found after TEST= 89 self.linenum = 1 # Output line number we are up to 90 self.in_section = None # Name of start...END section we are in 91 self.notes = [] # Series notes 92 self.section = [] # The current section...END section 93 self.series = series # Info about the patch series 94 self.is_log = is_log # True if indent like git log 95 self.keep_change_id = keep_change_id # True to keep Change-Id tags 96 self.in_change = None # Name of the change list we are in 97 self.change_version = 0 # Non-zero if we are in a change list 98 self.change_lines = [] # Lines of the current change 99 self.blank_count = 0 # Number of blank lines stored up 100 self.state = STATE_MSG_HEADER # What state are we in? 101 self.commit = None # Current commit 102 # List of unquoted test blocks, each a list of str lines 103 self.snippets = [] 104 self.cur_diff = None # Last 'diff' line seen (str) 105 self.cur_line = None # Last context (@@) line seen (str) 106 self.recent_diff = None # 'diff' line for current snippet (str) 107 self.recent_line = None # '@@' line for current snippet (str) 108 self.recent_quoted = collections.deque([], 5) 109 self.recent_unquoted = queue.Queue() 110 self.was_quoted = None 111 self.insert_base_commit = insert_base_commit 112 self.lines = [] # All lines in a commit message 113 self.msg = None # Full commit message including subject 114 115 @staticmethod 116 def process_text(text, is_comment=False): 117 """Process some text through this class using a default Commit/Series 118 119 Args: 120 text (str): Text to parse 121 is_comment (bool): True if this is a comment rather than a patch. 122 If True, PatchStream doesn't expect a patch subject at the 123 start, but jumps straight into the body 124 125 Returns: 126 PatchStream: object with results 127 """ 128 pstrm = PatchStream(Series()) 129 pstrm.commit = commit.Commit(None) 130 infd = io.StringIO(text) 131 outfd = io.StringIO() 132 if is_comment: 133 pstrm.state = STATE_PATCH_HEADER 134 pstrm.process_stream(infd, outfd) 135 return pstrm 136 137 def _add_warn(self, warn): 138 """Add a new warning to report to the user about the current commit 139 140 The new warning is added to the current commit if not already present. 141 142 Args: 143 warn (str): Warning to report 144 145 Raises: 146 ValueError: Warning is generated with no commit associated 147 """ 148 if not self.commit: 149 print('Warning outside commit: %s' % warn) 150 elif warn not in self.commit.warn: 151 self.commit.warn.append(warn) 152 153 def _add_to_series(self, line, name, value): 154 """Add a new Series-xxx tag. 155 156 When a Series-xxx tag is detected, we come here to record it, if we 157 are scanning a 'git log'. 158 159 Args: 160 line (str): Source line containing tag (useful for debug/error 161 messages) 162 name (str): Tag name (part after 'Series-') 163 value (str): Tag value (part after 'Series-xxx: ') 164 """ 165 if name == 'notes': 166 self.in_section = name 167 self.skip_blank = False 168 if self.is_log: 169 warn = self.series.AddTag(self.commit, line, name, value) 170 if warn: 171 self.commit.warn.append(warn) 172 173 def _add_to_commit(self, name): 174 """Add a new Commit-xxx tag. 175 176 When a Commit-xxx tag is detected, we come here to record it. 177 178 Args: 179 name (str): Tag name (part after 'Commit-') 180 """ 181 if name == 'notes': 182 self.in_section = 'commit-' + name 183 self.skip_blank = False 184 185 def _add_commit_rtag(self, rtag_type, who): 186 """Add a response tag to the current commit 187 188 Args: 189 rtag_type (str): rtag type (e.g. 'Reviewed-by') 190 who (str): Person who gave that rtag, e.g. 191 'Fred Bloggs <fred@bloggs.org>' 192 """ 193 self.commit.add_rtag(rtag_type, who) 194 195 def _close_commit(self, skip_last_line): 196 """Save the current commit into our commit list, and reset our state 197 198 Args: 199 skip_last_line (bool): True to omit the final line of self.lines 200 when building the commit message. This is normally the blank 201 line between two commits, except at the end of the log, where 202 there is no blank line 203 """ 204 if self.commit and self.is_log: 205 # Skip the blank line before the subject 206 lines = self.lines[:-1] if skip_last_line else self.lines 207 self.commit.msg = '\n'.join(lines[1:]) + '\n' 208 self.series.AddCommit(self.commit) 209 self.commit = None 210 self.lines = [] 211 # If 'END' is missing in a 'Cover-letter' section, and that section 212 # happens to show up at the very end of the commit message, this is 213 # the chance for us to fix it up. 214 if self.in_section == 'cover' and self.is_log: 215 self.series.cover = self.section 216 self.in_section = None 217 self.skip_blank = True 218 self.section = [] 219 220 self.cur_diff = None 221 self.recent_diff = None 222 self.recent_line = None 223 224 def _parse_version(self, value, line): 225 """Parse a version from a *-changes tag 226 227 Args: 228 value (str): Tag value (part after 'xxx-changes: ' 229 line (str): Source line containing tag 230 231 Returns: 232 int: The version as an integer 233 234 Raises: 235 ValueError: the value cannot be converted 236 """ 237 try: 238 return int(value) 239 except ValueError: 240 raise ValueError("%s: Cannot decode version info '%s'" % 241 (self.commit.hash, line)) 242 243 def _finalise_change(self): 244 """_finalise a (multi-line) change and add it to the series or commit""" 245 if not self.change_lines: 246 return 247 change = '\n'.join(self.change_lines) 248 249 if self.in_change == 'Series': 250 self.series.AddChange(self.change_version, self.commit, change) 251 elif self.in_change == 'Cover': 252 self.series.AddChange(self.change_version, None, change) 253 elif self.in_change == 'Commit': 254 self.commit.add_change(self.change_version, change) 255 self.change_lines = [] 256 257 def _finalise_snippet(self): 258 """Finish off a snippet and add it to the list 259 260 This is called when we get to the end of a snippet, i.e. the we enter 261 the next block of quoted text: 262 263 This is a comment from someone. 264 265 Something else 266 267 > Now we have some code <----- end of snippet 268 > more code 269 270 Now a comment about the above code 271 272 This adds the snippet to our list 273 """ 274 quoted_lines = [] 275 while self.recent_quoted: 276 quoted_lines.append(self.recent_quoted.popleft()) 277 unquoted_lines = [] 278 valid = False 279 while not self.recent_unquoted.empty(): 280 text = self.recent_unquoted.get() 281 if not (text.startswith('On ') and text.endswith('wrote:')): 282 unquoted_lines.append(text) 283 if text: 284 valid = True 285 if valid: 286 lines = [] 287 if self.recent_diff: 288 lines.append('> File: %s' % self.recent_diff) 289 if self.recent_line: 290 out = '> Line: %s / %s' % self.recent_line[:2] 291 if self.recent_line[2]: 292 out += ': %s' % self.recent_line[2] 293 lines.append(out) 294 lines += quoted_lines + unquoted_lines 295 if lines: 296 self.snippets.append(lines) 297 298 def process_line(self, line): 299 """Process a single line of a patch file or commit log 300 301 This process a line and returns a list of lines to output. The list 302 may be empty or may contain multiple output lines. 303 304 This is where all the complicated logic is located. The class's 305 state is used to move between different states and detect things 306 properly. 307 308 We can be in one of two modes: 309 self.is_log == True: This is 'git log' mode, where most output is 310 indented by 4 characters and we are scanning for tags 311 312 self.is_log == False: This is 'patch' mode, where we already have 313 all the tags, and are processing patches to remove junk we 314 don't want, and add things we think are required. 315 316 Args: 317 line (str): text line to process 318 319 Returns: 320 list: list of output lines, or [] if nothing should be output 321 322 Raises: 323 ValueError: a fatal error occurred while parsing, e.g. an END 324 without a starting tag, or two commits with two change IDs 325 """ 326 # Initially we have no output. Prepare the input line string 327 out = [] 328 line = line.rstrip('\n') 329 330 commit_match = RE_COMMIT.match(line) if self.is_log else None 331 332 if self.is_log: 333 if line[:4] == ' ': 334 line = line[4:] 335 336 # Handle state transition and skipping blank lines 337 series_tag_match = RE_SERIES_TAG.match(line) 338 change_id_match = RE_CHANGE_ID.match(line) 339 commit_tag_match = RE_COMMIT_TAG.match(line) 340 cover_match = RE_COVER.match(line) 341 signoff_match = RE_SIGNOFF.match(line) 342 leading_whitespace_match = RE_LEADING_WHITESPACE.match(line) 343 diff_match = RE_DIFF.match(line) 344 line_match = RE_LINE.match(line) 345 invalid_match = RE_INV_TAG.match(line) 346 tag_match = None 347 if self.state == STATE_PATCH_HEADER: 348 tag_match = RE_TAG.match(line) 349 is_blank = not line.strip() 350 if is_blank: 351 if (self.state == STATE_MSG_HEADER 352 or self.state == STATE_PATCH_SUBJECT): 353 self.state += 1 354 355 # We don't have a subject in the text stream of patch files 356 # It has its own line with a Subject: tag 357 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 358 self.state += 1 359 elif commit_match: 360 self.state = STATE_MSG_HEADER 361 if self.state != STATE_MSG_HEADER: 362 self.lines.append(line) 363 364 # If a tag is detected, or a new commit starts 365 if series_tag_match or commit_tag_match or change_id_match or \ 366 cover_match or signoff_match or self.state == STATE_MSG_HEADER: 367 # but we are already in a section, this means 'END' is missing 368 # for that section, fix it up. 369 if self.in_section: 370 self._add_warn("Missing 'END' in section '%s'" % self.in_section) 371 if self.in_section == 'cover': 372 self.series.cover = self.section 373 elif self.in_section == 'notes': 374 if self.is_log: 375 self.series.notes += self.section 376 elif self.in_section == 'commit-notes': 377 if self.is_log: 378 self.commit.notes += self.section 379 else: 380 # This should not happen 381 raise ValueError("Unknown section '%s'" % self.in_section) 382 self.in_section = None 383 self.skip_blank = True 384 self.section = [] 385 # but we are already in a change list, that means a blank line 386 # is missing, fix it up. 387 if self.in_change: 388 self._add_warn("Missing 'blank line' in section '%s-changes'" % 389 self.in_change) 390 self._finalise_change() 391 self.in_change = None 392 self.change_version = 0 393 394 # If we are in a section, keep collecting lines until we see END 395 if self.in_section: 396 if line == 'END': 397 if self.in_section == 'cover': 398 self.series.cover = self.section 399 elif self.in_section == 'notes': 400 if self.is_log: 401 self.series.notes += self.section 402 elif self.in_section == 'commit-notes': 403 if self.is_log: 404 self.commit.notes += self.section 405 else: 406 # This should not happen 407 raise ValueError("Unknown section '%s'" % self.in_section) 408 self.in_section = None 409 self.skip_blank = True 410 self.section = [] 411 else: 412 self.section.append(line) 413 414 # If we are not in a section, it is an unexpected END 415 elif line == 'END': 416 raise ValueError("'END' wihout section") 417 418 # Detect the commit subject 419 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 420 self.commit.subject = line 421 422 # Detect the tags we want to remove, and skip blank lines 423 elif RE_REMOVE.match(line) and not commit_tag_match: 424 self.skip_blank = True 425 426 # TEST= should be the last thing in the commit, so remove 427 # everything after it 428 if line.startswith('TEST='): 429 self.found_test = True 430 elif self.skip_blank and is_blank: 431 self.skip_blank = False 432 433 # Detect Cover-xxx tags 434 elif cover_match: 435 name = cover_match.group(1) 436 value = cover_match.group(2) 437 if name == 'letter': 438 self.in_section = 'cover' 439 self.skip_blank = False 440 elif name == 'letter-cc': 441 self._add_to_series(line, 'cover-cc', value) 442 elif name == 'changes': 443 self.in_change = 'Cover' 444 self.change_version = self._parse_version(value, line) 445 446 # If we are in a change list, key collected lines until a blank one 447 elif self.in_change: 448 if is_blank: 449 # Blank line ends this change list 450 self._finalise_change() 451 self.in_change = None 452 self.change_version = 0 453 elif line == '---': 454 self._finalise_change() 455 self.in_change = None 456 self.change_version = 0 457 out = self.process_line(line) 458 elif self.is_log: 459 if not leading_whitespace_match: 460 self._finalise_change() 461 self.change_lines.append(line) 462 self.skip_blank = False 463 464 # Detect Series-xxx tags 465 elif series_tag_match: 466 name = series_tag_match.group(1) 467 value = series_tag_match.group(2) 468 if name == 'changes': 469 # value is the version number: e.g. 1, or 2 470 self.in_change = 'Series' 471 self.change_version = self._parse_version(value, line) 472 else: 473 self._add_to_series(line, name, value) 474 self.skip_blank = True 475 476 # Detect Change-Id tags 477 elif change_id_match: 478 if self.keep_change_id: 479 out = [line] 480 value = change_id_match.group(1) 481 if self.is_log: 482 if self.commit.change_id: 483 raise ValueError( 484 "%s: Two Change-Ids: '%s' vs. '%s'" % 485 (self.commit.hash, self.commit.change_id, value)) 486 self.commit.change_id = value 487 self.skip_blank = True 488 489 # Detect Commit-xxx tags 490 elif commit_tag_match: 491 name = commit_tag_match.group(1) 492 value = commit_tag_match.group(2) 493 if name == 'notes': 494 self._add_to_commit(name) 495 self.skip_blank = True 496 elif name == 'changes': 497 self.in_change = 'Commit' 498 self.change_version = self._parse_version(value, line) 499 elif name == 'cc': 500 self.commit.add_cc(value.split(',')) 501 elif name == 'added-in': 502 version = self._parse_version(value, line) 503 self.commit.add_change(version, '- New') 504 self.series.AddChange(version, None, '- %s' % 505 self.commit.subject) 506 else: 507 self._add_warn('Line %d: Ignoring Commit-%s' % 508 (self.linenum, name)) 509 510 # Detect invalid tags 511 elif invalid_match: 512 raise ValueError("Line %d: Invalid tag = '%s'" % 513 (self.linenum, line)) 514 515 # Detect the start of a new commit 516 elif commit_match: 517 self._close_commit(True) 518 self.commit = commit.Commit(commit_match.group(1)) 519 520 # Detect tags in the commit message 521 elif tag_match: 522 rtag_type, who = tag_match.groups() 523 self._add_commit_rtag(rtag_type, who) 524 # Remove Tested-by self, since few will take much notice 525 if (rtag_type == 'Tested-by' and 526 who.find(os.getenv('USER') + '@') != -1): 527 self._add_warn("Ignoring '%s'" % line) 528 elif rtag_type == 'Patch-cc': 529 self.commit.add_cc(who.split(',')) 530 else: 531 out = [line] 532 533 # Suppress duplicate signoffs 534 elif signoff_match: 535 if (self.is_log or not self.commit or 536 self.commit.check_duplicate_signoff(signoff_match.group(1))): 537 out = [line] 538 539 # Well that means this is an ordinary line 540 else: 541 # Look for space before tab 542 mat = RE_SPACE_BEFORE_TAB.match(line) 543 if mat: 544 self._add_warn('Line %d/%d has space before tab' % 545 (self.linenum, mat.start())) 546 547 # OK, we have a valid non-blank line 548 out = [line] 549 self.linenum += 1 550 self.skip_blank = False 551 552 if diff_match: 553 self.cur_diff = diff_match.group(1) 554 555 # If this is quoted, keep recent lines 556 if not diff_match and self.linenum > 1 and line: 557 if line.startswith('>'): 558 if not self.was_quoted: 559 self._finalise_snippet() 560 self.recent_line = None 561 if not line_match: 562 self.recent_quoted.append(line) 563 self.was_quoted = True 564 self.recent_diff = self.cur_diff 565 else: 566 self.recent_unquoted.put(line) 567 self.was_quoted = False 568 569 if line_match: 570 self.recent_line = line_match.groups() 571 572 if self.state == STATE_DIFFS: 573 pass 574 575 # If this is the start of the diffs section, emit our tags and 576 # change log 577 elif line == '---': 578 self.state = STATE_DIFFS 579 580 # Output the tags (signoff first), then change list 581 out = [] 582 log = self.series.MakeChangeLog(self.commit) 583 out += [line] 584 if self.commit: 585 out += self.commit.notes 586 out += [''] + log 587 elif self.found_test: 588 if not RE_ALLOWED_AFTER_TEST.match(line): 589 self.lines_after_test += 1 590 591 return out 592 593 def finalise(self): 594 """Close out processing of this patch stream""" 595 self._finalise_snippet() 596 self._finalise_change() 597 self._close_commit(False) 598 if self.lines_after_test: 599 self._add_warn('Found %d lines after TEST=' % self.lines_after_test) 600 601 def _write_message_id(self, outfd): 602 """Write the Message-Id into the output. 603 604 This is based on the Change-Id in the original patch, the version, 605 and the prefix. 606 607 Args: 608 outfd (io.IOBase): Output stream file object 609 """ 610 if not self.commit.change_id: 611 return 612 613 # If the count is -1 we're testing, so use a fixed time 614 if self.commit.count == -1: 615 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59) 616 else: 617 time_now = datetime.datetime.now() 618 619 # In theory there is email.utils.make_msgid() which would be nice 620 # to use, but it already produces something way too long and thus 621 # will produce ugly commit lines if someone throws this into 622 # a "Link:" tag in the final commit. So (sigh) roll our own. 623 624 # Start with the time; presumably we wouldn't send the same series 625 # with the same Change-Id at the exact same second. 626 parts = [time_now.strftime("%Y%m%d%H%M%S")] 627 628 # These seem like they would be nice to include. 629 if 'prefix' in self.series: 630 parts.append(self.series['prefix']) 631 if 'postfix' in self.series: 632 parts.append(self.series['postfix']) 633 if 'version' in self.series: 634 parts.append("v%s" % self.series['version']) 635 636 parts.append(str(self.commit.count + 1)) 637 638 # The Change-Id must be last, right before the @ 639 parts.append(self.commit.change_id) 640 641 # Join parts together with "." and write it out. 642 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts)) 643 644 def process_stream(self, infd, outfd): 645 """Copy a stream from infd to outfd, filtering out unwanting things. 646 647 This is used to process patch files one at a time. 648 649 Args: 650 infd (io.IOBase): Input stream file object 651 outfd (io.IOBase): Output stream file object 652 """ 653 # Extract the filename from each diff, for nice warnings 654 fname = None 655 last_fname = None 656 re_fname = re.compile('diff --git a/(.*) b/.*') 657 658 self._write_message_id(outfd) 659 660 while True: 661 line = infd.readline() 662 if not line: 663 break 664 out = self.process_line(line) 665 666 # Try to detect blank lines at EOF 667 for line in out: 668 match = re_fname.match(line) 669 if match: 670 last_fname = fname 671 fname = match.group(1) 672 if line == '+': 673 self.blank_count += 1 674 else: 675 if self.blank_count and (line == '-- ' or match): 676 self._add_warn("Found possible blank line(s) at end of file '%s'" % 677 last_fname) 678 outfd.write('+\n' * self.blank_count) 679 outfd.write(line + '\n') 680 self.blank_count = 0 681 self.finalise() 682 if self.insert_base_commit: 683 if self.series.base_commit: 684 print(f'base-commit: {self.series.base_commit.hash}', 685 file=outfd) 686 if self.series.branch: 687 print(f'branch: {self.series.branch}', file=outfd) 688 689 690def insert_tags(msg, tags_to_emit): 691 """Add extra tags to a commit message 692 693 The tags are added after an existing block of tags if found, otherwise at 694 the end. 695 696 Args: 697 msg (str): Commit message 698 tags_to_emit (list): List of tags to emit, each a str 699 700 Returns: 701 (str) new message 702 """ 703 out = [] 704 done = False 705 emit_tags = False 706 emit_blank = False 707 for line in msg.splitlines(): 708 if not done: 709 signoff_match = RE_SIGNOFF.match(line) 710 tag_match = RE_TAG.match(line) 711 if tag_match or signoff_match: 712 emit_tags = True 713 if emit_tags and not tag_match and not signoff_match: 714 out += tags_to_emit 715 emit_tags = False 716 done = True 717 emit_blank = not (signoff_match or tag_match) 718 else: 719 emit_blank = line 720 out.append(line) 721 if not done: 722 if emit_blank: 723 out.append('') 724 out += tags_to_emit 725 return '\n'.join(out) 726 727def get_list(commit_range, git_dir=None, count=None): 728 """Get a log of a list of comments 729 730 This returns the output of 'git log' for the selected commits 731 732 Args: 733 commit_range (str): Range of commits to count (e.g. 'HEAD..base') 734 git_dir (str): Path to git repositiory (None to use default) 735 count (int): Number of commits to list, or None for no limit 736 737 Returns 738 str: String containing the contents of the git log 739 """ 740 params = gitutil.log_cmd(commit_range, reverse=True, count=count, 741 git_dir=git_dir) 742 return command.run_one(*params, capture=True).stdout 743 744def get_metadata_for_list(commit_range, git_dir=None, count=None, 745 series=None, allow_overwrite=False): 746 """Reads out patch series metadata from the commits 747 748 This does a 'git log' on the relevant commits and pulls out the tags we 749 are interested in. 750 751 Args: 752 commit_range (str): Range of commits to count (e.g. 'HEAD..base') 753 git_dir (str): Path to git repositiory (None to use default) 754 count (int): Number of commits to list, or None for no limit 755 series (Series): Object to add information into. By default a new series 756 is started. 757 allow_overwrite (bool): Allow tags to overwrite an existing tag 758 759 Returns: 760 Series: Object containing information about the commits. 761 """ 762 if not series: 763 series = Series() 764 series.allow_overwrite = allow_overwrite 765 stdout = get_list(commit_range, git_dir, count) 766 pst = PatchStream(series, is_log=True) 767 for line in stdout.splitlines(): 768 pst.process_line(line) 769 pst.finalise() 770 return series 771 772def get_metadata(branch, start, count, git_dir=None): 773 """Reads out patch series metadata from the commits 774 775 This does a 'git log' on the relevant commits and pulls out the tags we 776 are interested in. 777 778 Args: 779 branch (str): Branch to use (None for current branch) 780 start (int): Commit to start from: 0=branch HEAD, 1=next one, etc. 781 count (int): Number of commits to list 782 783 Returns: 784 Series: Object containing information about the commits. 785 """ 786 top = f"{branch if branch else 'HEAD'}~{start}" 787 series = get_metadata_for_list(top, git_dir, count) 788 series.base_commit = commit.Commit( 789 gitutil.get_hash(f'{top}~{count}', git_dir)) 790 series.branch = branch or gitutil.get_branch() 791 series.top = top 792 return series 793 794def get_metadata_for_test(text): 795 """Process metadata from a file containing a git log. Used for tests 796 797 Args: 798 text: 799 800 Returns: 801 Series: Object containing information about the commits. 802 """ 803 series = Series() 804 pst = PatchStream(series, is_log=True) 805 for line in text.splitlines(): 806 pst.process_line(line) 807 pst.finalise() 808 return series 809 810def fix_patch(backup_dir, fname, series, cmt, keep_change_id=False, 811 insert_base_commit=False, cwd=None): 812 """Fix up a patch file, by adding/removing as required. 813 814 We remove our tags from the patch file, insert changes lists, etc. 815 The patch file is processed in place, and overwritten. 816 817 A backup file is put into backup_dir (if not None). 818 819 Args: 820 backup_dir (str): Path to directory to use to backup the file 821 fname (str): Filename to patch file to process 822 series (Series): Series information about this patch set 823 cmt (Commit): Commit object for this patch file 824 keep_change_id (bool): Keep the Change-Id tag. 825 insert_base_commit (bool): True to add the base commit to the end 826 cwd (str): Directory containing filename, or None for current 827 828 Return: 829 list: A list of errors, each str, or [] if all ok. 830 """ 831 fname = os.path.join(cwd or '', fname) 832 handle, tmpname = tempfile.mkstemp() 833 outfd = os.fdopen(handle, 'w', encoding='utf-8') 834 infd = open(fname, 'r', encoding='utf-8') 835 pst = PatchStream(series, keep_change_id=keep_change_id, 836 insert_base_commit=insert_base_commit) 837 pst.commit = cmt 838 pst.process_stream(infd, outfd) 839 infd.close() 840 outfd.close() 841 842 # Create a backup file if required 843 if backup_dir: 844 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 845 shutil.move(tmpname, fname) 846 return cmt.warn 847 848def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False, 849 cwd=None): 850 """Fix up a list of patches identified by filenames 851 852 The patch files are processed in place, and overwritten. 853 854 Args: 855 series (Series): The Series object 856 fnames (:type: list of str): List of patch files to process 857 keep_change_id (bool): Keep the Change-Id tag. 858 insert_base_commit (bool): True to add the base commit to the end 859 cwd (str): Directory containing the patch files, or None for current 860 """ 861 # Current workflow creates patches, so we shouldn't need a backup 862 backup_dir = None #tempfile.mkdtemp('clean-patch') 863 count = 0 864 for fname in fnames: 865 cmt = series.commits[count] 866 cmt.patch = fname 867 cmt.count = count 868 result = fix_patch(backup_dir, fname, series, cmt, 869 keep_change_id=keep_change_id, 870 insert_base_commit=insert_base_commit, cwd=cwd) 871 if result: 872 print('%d warning%s for %s:' % 873 (len(result), 's' if len(result) > 1 else '', fname)) 874 for warn in result: 875 print('\t%s' % warn) 876 print() 877 count += 1 878 print('Cleaned %d patch%s' % (count, 'es' if count > 1 else '')) 879 880def insert_cover_letter(fname, series, count, cwd=None): 881 """Inserts a cover letter with the required info into patch 0 882 883 Args: 884 fname (str): Input / output filename of the cover letter file 885 series (Series): Series object 886 count (int): Number of patches in the series 887 cwd (str): Directory containing filename, or None for current 888 """ 889 fname = os.path.join(cwd or '', fname) 890 fil = open(fname, 'r') 891 lines = fil.readlines() 892 fil.close() 893 894 fil = open(fname, 'w') 895 text = series.cover 896 prefix = series.GetPatchPrefix() 897 for line in lines: 898 if line.startswith('Subject:'): 899 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc 900 zero_repeat = int(math.log10(count)) + 1 901 zero = '0' * zero_repeat 902 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0]) 903 904 # Insert our cover letter 905 elif line.startswith('*** BLURB HERE ***'): 906 # First the blurb test 907 line = '\n'.join(text[1:]) + '\n' 908 if series.get('notes'): 909 line += '\n'.join(series.notes) + '\n' 910 911 # Now the change list 912 out = series.MakeChangeLog(None) 913 line += '\n' + '\n'.join(out) 914 fil.write(line) 915 916 # Insert the base commit and branch 917 if series.base_commit: 918 print(f'base-commit: {series.base_commit.hash}', file=fil) 919 if series.branch: 920 print(f'branch: {series.branch}', file=fil) 921 922 fil.close() 923