#!/usr/local/bin/python3.10
import glob, json, subprocess, time, optparse, fnmatch, cmd, os, re
from xml.etree import ElementTree

'''
Program to help command-line users have better access to IETF-related documents
   See help text below.

Style note:  The programming style here is, well, um, idiomatic at best. A better way to say this
              is that I am now embarrassed by the style. I have already fixed a bunch of bad stylistic
              choices I made, but it is still a tad silly. I apologize for CamelCase variables and
              the weird use of global variables, but at least the program is still readable.
'''
__version__ = "1.23"
__license__ = "https://en.wikipedia.org/wiki/WTFPL"

# Version history:
#  1.0
#    Initial release
#  1.1
#    Added bcp
#    Split out the command-calling in the CLI to make it clearer
#  1.2
#    Added more places to look for the config file
#    Fixed help text for "charter" to make wildcard use clearer
#    Added auth48
#  1.3
#    Added the WTFPL license
#    Added "rfcextra " which opens RFCs that replace the requested RFC and the errata page for the
#       requested RFC if the database says there is errata
#    Added "rfcstatus" which lists RFC status from the RFC Editor's database
#    Added "draftstatus" which lists the I-D status from the Datatracer
#    Added "author" command which lists all drafts and RFCs by an author
#    Added parsing the Datatracker's I-D database during mirror; file is saved as a local JSON database
#    Added parsing the RFC Editor's database during mirror; file is saved as a local JSON database
#    Added the --quiet command line option
#    Made "tracker" also work for RFCs
#    Made the help a bit more helpful
#    Fixed bug that found too many drafts in in-notes/authors
#    Fixed bug so that the help text fit in 80 column windows
#    Fixed bug for finding the configuration file
#  1.4
#    Added "iesg docs" and "iesg agenda"
#    Fixed bug with displaying multiple drafts that have the same name beginnings
#  1.5
#	   Made changes to get new-style charters with "charter-new"
#  1.6
#    Added "charternew"
#    Added "conflict"
#  1.7
#    Got rid of "charter" because new WGs don't work with old charter
#  1.8
#    Added .encode('utf-8') to author and AD names in draftstatus because those might be UTF-8
#       Clearly, this needs to be dealt with better in a future version
#  1.9
#    Added "rfcinfo", and changed "tools" to only go to the tools.ietf.org site
#  1.10
#    Changed the rsync targets from www.ietf.org to rsync.ietf.org
#  1.11
#    Mirror conflict reviews and status changes when updating, but don't expose them in the UI
#  1.12
#    Changed all http: URIs to https:
#  1.13
#    Fixed ietf-rfc-status.json generation, and fixed a minor typo
#  1.14
#    Changed the location of the IESG directory, which had been broken for a while
#  1.15
#    Lower-cased the inputs to match arguments better
#    Caused a bug in the AUTH48 HTMLs to not cause the program to die
#  1.16
#    Added "std" command
#    Added "rg" command
#  1.17
#    Required Python 3 (which removed need for encode/decode from version 1.8)
#    Had "mirror" collect "Updates:" and "Obsoletes:" information from all drafts for the database
#    Made "rfcextra" non-recursive to prevent massive openings for highly-updated RFCs
#    Made "rfc" more verbose by listing RFC titles with the returned information
#    Removed "rfcstatus" and "draftstatus" because the information is now in "rfc" and "draft"
#    Added information to "rfc" of drafts that might be updating or obsoleting the RFC
#    Added information to "draft" of which RFCs the draft is updating and/or obsoleting
#    Made "bcp" and "std" open the RFCs, not the combinded files; also give details on the CLI
#    Added "bcponly" and "stdonly" to open the combined files
#    Added "draftreport" to list all drafts updating and obsoleting RFCs, by draft name and by RFC
#    Simplified opening of the JSON files
#  1.18
#    Added a check in "draftreport" to show the files that had non-parseable "Updates" and "Obsoletes"
#  1.19
#    Dropped "ftp." from the RFC Editor's rsync server name at the request of AMS
#  1.20
#    Mirror the draft XML files
#    Allow showing HTML and XML files in the "draft" command with a new first argument
#  1.21
#    Small fixes to determining which drafts presumably update RFCs
#  1.22
#    If all_id2.txt does not have some draft that is on disk, skip over it
#  1.23
#    Skip if a draft does not appear in the database


##########
# Utility functions and definitions
##########

KnownCmds = ("auth48", "author", "bcp", "bcponly", "charter", "conflict", "diff", "draft", "draftreport", \
	 "iesg", "mirror", "rfc", "rfcextra", "rfcinfo", "rg", "std", "stdonly", "tools", "tracker", "foo")
ConfigPlaces = ("~/.ietf/ietf.config", "/etc/ietf.config")

# Make a block of text that can be executed in the CLI
CLICmdCode = ""
for ThisCmd in KnownCmds:
	ThisCode = '''
def do_REPLTHISCMD(self, RestOfArgs):
  Cmd_REPLTHISCMD(RestOfArgs.split(' '))
def help_REPLTHISCMD(self):
  CheckHelp('REPLTHISCMD', '__helptext__')
'''
	CLICmdCode += ThisCode.replace("REPLTHISCMD", ThisCmd)

# Strip leading zeros for RFC numbers
def StripLeadingZeros(InStr):
	return(re.sub(re.compile(r'^0+(.*)'), "\\1", InStr))

# Find a draft in the in-notes/authors directory, return "rfc1234" or ""
def FindDraftInAuth48(basename):
	TheDiffs = glob.glob(os.path.join(RFCDir, "authors", "*-diff.html"))
	for ThisDiff in TheDiffs:
		try:
			InTextLines = open(ThisDiff, mode="r").readlines()
		except:
			exit("Weird: could not read '" + ThisDiff + "' even though it exists. Exit.")
		for InText in InTextLines[0:40]:
			### The following try/except is needed due to erroneous non-ASCII text in some of these HTML files
			try:
				if InText.find("<strike><font color='red'>" + basename) > -1:
					return(ThisDiff.replace(os.path.join(RFCDir, "authors", ""), "").replace("-diff.html", ""))
			except:
				pass
	return("")  # Only here if there was no file in AUTH48

# Open a URL in the browser, but give a warning in the terminal if the command is "less"
def WebDisplay(TheURL, TheArg):
	TheRet = os.system(DisplayWebCommand + TheURL + TheArg)
	if TheRet > 0:
		print("The command used to display web content, '" + DisplayWebCommand \
		 + TheURL + TheArg + "', had an error.'")
	if DisplayWebCommand == "less ":
		print("The reason that this HTML was displayed on your console is that you do not have\n" \
			"'DisplayWebCommand' defined in the file '" + ConfigFile + "'.")

# Create a command-line processor for our commands
class OurCLI(cmd.Cmd):
	intro = "Command line processor for ietf commands; try 'help' for more info."
	prompt = "ietf: "
	# Make just pressing Return not do anything
	def emptyline(self):
		pass
	# Make it easy to exit
	def do_exit(self, RestOfArgs):
		return True
	do_quit = do_q = do_exit
	def do_EOF(self, RestOfArgs):
		print()
		return True
	def default(self, RestOfArgs):
		print("Unknown command '" + RestOfArgs + "'. Try 'help' for a list of commands.")
	# Let them do shell commands
	def do_shell(self, RestOfArgs):
		print("Execuiting shell command: '" + RestOfArgs + "'")
		os.system(RestOfArgs)
	# Fill in the needed definitions for all the known commands
	#   This was created as CLICmdCode above
	exec(CLICmdCode)
	# Do our own help
	def do_help(self, RestOfArgs):
		if RestOfArgs in KnownCmds:
			CheckHelp(RestOfArgs, "__helptext__")
		else:
			CheckHelp("allclicmds", "__helptext__")
	# Allow to change commandline settings
	def do_tombstones(self, RestOfArgs):
		global DisplayTombstones
		DisplayTombstones = True
	def do_maxdrafts(self, RestOfArgs):
		try:
			global MaxDrafts
			MaxDrafts = int(RestOfArgs)
		except:
			exit("The argument to 'maxdrafts' must be a positive integer. Exiting.")
	def do_usedraftnumbers(self, RestOfArgs):
		global UseDraftNumbers
		UseDraftNumbers = True
	def do_quiet(self, RestOfArgs):
		global QuietDraft
		QuietDraft = True

def ListIDsAndNames(ListOfIDs):
	# Returns a string containing indented RFC numbers and names
	RetString = ""
	for ThisIDName in sorted(ListOfIDs):
		if IDStatusDB.get(ThisIDName):
			RetString += f"    {ThisIDName}: {IDStatusDB[ThisIDName]['title']}\n"
		else:
			RetString += f"    {ThisIDName}\n"
	return RetString.rstrip()

def ListRFCsAndNames(ListOfRFCs):
	# Returns a string containing indented RFC numbers and names
	RetString = ""
	for ThisRFCNum in sorted(ListOfRFCs):
		if RFCStatusDB.get(ThisRFCNum):
			RetString += f"    {ThisRFCNum}: {RFCStatusDB[ThisRFCNum]['title']}\n"
		else:
			RetString += f"    {ThisRFCNum}\n"
	return RetString.rstrip()

# Print help text if this is called with no args or with a single arg of "__helptext__"
#   All commands other than "mirror" and "draftreport" need args.
def CheckHelp(TheCaller, InArgs):
	if ((InArgs == "__helptext__") or ((InArgs == []) and (TheCaller not in ("mirror", "draftreport")))):
		if HelpText.get(TheCaller, "") != "":
			print(HelpText[TheCaller])
		else:
			print("No help text available for '" + TheCaller + "'.")
		return True
	else:
		return False

HelpText = {
	"auth48": '''auth48:
    Takes a list of RFC numbers or draft names, determines if there are AUTH48
    files associated with them, and displays the various files.''',
	"bcp": '''bcp:
    Takes a list of BCP numbers. Displays the RFC in the BCPs using the text
    dispay program.''',
	"bcponly": '''bcponly:
    Takes a list of BCP numbers. Displays the combined BCP documentss
    using the text dispay program. You can also give 'index' as an argument
     to see bcp-index.txt.''',
	"charter": '''charter:
    Takes a list of WG names. Displays the charter for each WG using the text
    dispay program. Wildcards are appended to the beginning and end of the
    charter name given, and can also be given in the name. The charters are
    gotten from the new-style charters in the "charter" directory, which was
    begun in June 2012.''',
  "conflict": '''conflict:
    Takes a draft name (with or without the '-nn' version number or '.txt''
    and displays the HTML conflict review, if it exists.''',
	"diff": '''diff:
    Takes a draft name (with or without the '-nn' version number or '.txt'
    and displays the HTML diff between it and the preceding version on the
    IETF Tools page using your web display program.''',
	"draft": '''draft:
    Takes a list of draft file names. Displays the drafts found using the text
    dispay program. Substrings can be used instead of full names. There are 
    command-line options to change the way this shows tombstones (where a
    draft has expired or been replaced with an RFC). Give 'html' or 'xml' as
    the first argument to display the HTML or XML version, if such
    a file is available. You can also give 'abstracts' as an argument to see
    1id-abstracts.txt. This command also reports the status from the
    Datatracker database for each draft listed.''',
	"draftreport": '''draftreport:
    Prints out a report listing lots of things about the drafts, particularly
    about drafts that update and obsolete RFCs.''',
  "iesg": '''iesg:
    Displays the next agenda (when given the "agenda" argument") or the list
    of documents under consideration (when given the "docs" argument) in the
    web display program''',
	"mirror": '''mirror:
    Updates your local mirror of IETF directories, such as all drafts, RFCs,
    and WG charters.''',
	"rfc": '''rfc:
    Takes a list of RFC file names. Displays the RFCs found using the text
    dispay program. You do not need to give 'rfc' or '.txt' in the file
    names. You can also give 'index' as an argument to see rfc-index.txt.
    This command searches both the main RFC directory and the pre-publication
    (AUTH48) directory. Also lists information about the RFC take from
    various databases.''',
	"rfcextra": '''rfcextra:
    Similar to 'rfc' but opens additional files. It will automatically open
    RFCs that obsolete and update the one given, and will open errata in the
    browser if the RFC Editor's database indicates that such errata exists.''',
	"rfcinfo": '''rfcinfo:
    Takes a list of RFC numbers and opens the info pages from the RFC Editor's
    web site''',
	"rg": '''rg:
    Takes a list of IRTF RG names. Displays the result from the IETF Datatracker
    pages in the web dispay program. RG names are matched exactly.''',
	"std": '''std:
    Takes a list of STD numbers. Displays the STD RFCs found using the text
    dispay program. Note that this might be a single text file that has
    more than one RFC concatenated. You can also give 'index' as an argument
    to see std-index.txt.''',
	"tools": '''tools:
    Takes a list of draft file names, RFC names, and/or WG names. Displays the
    result from the IETF Tools pages in the web dispay program. Draft names
    can be either complete or be missing the '-nn' version number and '.txt'.
    RFC names can be given as 'rfc1234' or '1234'. WG names are matched
    exactly.''',
	"tracker": '''tracker:
    Takes a list of draft file names and/or WG names. Displays the
    result from the IETF Datatracker pages in the web dispay program. Draft
    names and WG names are matched exactly. For IRTF RGs, use the 'rg'
    command.'''
}
AllHelp = "Command-line interface for displaying IETF-related information. Version " \
	+ __version__ + ".\nCommands are:\n"
for ThisHelp in sorted(HelpText.keys()):
	AllHelp += " " + HelpText[ThisHelp] + "\n"
ArgsCLIHelp = "You can cause tombstone drafts to be displayed in the 'draft' command\n" \
	+ "    by giving the 'tombstones' command by itself.\n" \
	+ "You can increase the number of drafts that will be opened by the 'draft'\n" \
	+ "    command by giving the 'maxdrafts' command followed by an integer.\n" \
	+ "You can require that the 'draft' command only use full draft names\n" \
	+ "    (including draft numbers and '.txt') by giving the 'usedraftnumbers'\n" \
	+ "    command by itself.\n" \
	+ "You can make the 'draft' command not tell you about tombstones by giving\n" \
	+ "    the 'quiet' command by itself.\n" 
AllCLIHelp = AllHelp + ArgsCLIHelp \
	+ "There is also a 'shell' command to give shell commands from within\n" \
	+ "    this processor.\n" \
	+ "Use 'q' or 'quit' or 'exit' to leave the program."
ArgsShellHelp = "You can cause tombstone drafts to be displayed in the 'draft' command\n" \
	+ "    with the --tombstones argument.\n" \
	+ "You can increase the number of drafts that will be opened by the 'draft'\n" \
	+ "    command with the --maxdrafts= argument followed by an integer.\n" \
	+ "You can require that the 'draft' command only use full draft names\n" \
	+ "    (including draft numbers and '.txt') with the --usedraftnumbers'\n" \
	+ "    argument.\n" \
	+ "You can make the 'draft' command not tell you about tombstones with the\n" \
	+ "    --quiet argument.\n" 
AllShellHelp = AllHelp + ArgsShellHelp
HelpText["allclicmds"] = AllCLIHelp
HelpText["allshellcmds"] = AllShellHelp

##########
# The commands themselves
##########

### auth48 -- Open all appropriate files for a doc in AUTH48
def Cmd_auth48(Args):
	if CheckHelp("auth48", Args): return
	if Args[0] == "":
		print("Must give at least one draft name or RFC name; skipping.")
		return
	def ShowAuth48s(RFCfile):
		# Incoming file is in format "rfc1234"
		# Open the text file
		os.system(DisplayTextCommand + os.path.join(RFCDir, "authors", RFCfile + ".txt"))
		# Open the local diff in the browser 
		WebDisplay("file:///", os.path.join(RFCDir, "authors", RFCfile + "-diff.html"))
		# Show the status on the RFC Editor's site
		WebDisplay("https://www.rfc-editor.org/auth48/", RFCfile)
	for ThisArg in Args:
		# If it is just a number, check for the RFC
		if ThisArg.isdigit():
			if os.path.exists(os.path.join(RFCDir, "authors", "rfc" + ThisArg + ".txt")):
				ShowAuth48s("rfc" + ThisArg)
			else:
				print("You specified an all-digit argument, '" + ThisArg + "', but a corresponding RFC doesn't " \
					+ "exist in the AUTH48 directory. Skipping.")
		elif ((ThisArg[0:3] == "rfc") and (ThisArg[3:7].isdigit())):
			if os.path.exists(os.path.join(RFCDir, "authors", "rfc" + ThisArg[3:7] + ".txt")):
				ShowAuth48s("rfc" + ThisArg[3:7])
			else:
				print("You specified 'rfc' and some digits, but a corresponding RFC doesn't " \
					+ "exist in the AUTH48 directory. Skipping.")
		elif ThisArg.startswith("draft-"):
			ThisBaseName = os.path.basename(ThisArg)
			ThisAuth48 = FindDraftInAuth48(ThisBaseName)
			if ThisAuth48 != "":
				ShowAuth48s(ThisAuth48)
			else:
				print("You gave a draft name, but that draft doesn't have an AUTH48 RFC associated with it. Skipping.")
		else:
			print("Didn't recognize the argument '" + ThisArg + "'. Skipping.")

### author -- Search for drafts and RFCs with a particular author
def Cmd_author(Args):
	if CheckHelp("author", Args): return
	if Args[0] == "":
		print("Must give at least one string to search for; skipping.")
		return
	for ThisArg in Args:
		FoundRFCs = []
		FoundIDs = []
		for ThisRFC in sorted(RFCStatusDB.keys()):
			if re.search(".*" + ThisArg + ".*", str(RFCStatusDB[ThisRFC]["authors"])):
				FoundRFCs.append(ThisRFC)
		if FoundRFCs:
			print("Found '" + ThisArg + "' as author in RFCs:")
			for ThisFoundRFC in FoundRFCs:
				print("  RFC " + ThisFoundRFC + "  " + RFCStatusDB[ThisFoundRFC]["title"])
		for ThisID in sorted(IDStatusDB.keys()):
			if re.search(".*" + ThisArg + ".*", repr(IDStatusDB[ThisID]["authors"])):
				FoundIDs.append(ThisID)
		if FoundIDs:
			print("Found '" + ThisArg + "' as author in IDs:")
			for ThisFoundID in FoundIDs:
				print("  " + ThisFoundID + "  " + IDStatusDB[ThisFoundID]["title"])
			
### bcp -- Open THe RFCs in a BCP locally
def Cmd_bcp(Args):
	if CheckHelp("bcp", Args): return
	if Args[0] == "":
		print("Must give at least one BCP number or 'index'; skipping.")
		return
	for ThisArg in Args:
		StrippedBCP = StripLeadingZeros(ThisArg)
		# Print out the relevant material from the BCPStatusDB
		ThisStatus = BCPStatusDB.get(StrippedBCP)
		if ThisStatus:
			if BCPStatusDB[StrippedBCP].get("is-also"):
				print("RFCs in BCP:" + StrippedBCP)
				for ThisRFC in BCPStatusDB[StrippedBCP]["is-also"]:
					print(ListRFCsAndNames([ThisRFC]))
					FullPathToRFC = (os.path.join(RFCDir, "rfc" + ThisRFC + ".txt"))
					if os.path.exists(FullPathToRFC):
						os.system(DisplayTextCommand + FullPathToRFC)
					else:
						print("Could not find " + FullPathToRFC)
			else:
				print("The database says that there are no RFCs in BCP" + StrippedBCP)
		else:
			print("Did not find any status information in the BCP status database")

### bcponly -- Open BCP documents locally
def Cmd_bcponly(Args):
	if CheckHelp("bcponly", Args): return
	if Args[0] == "":
		print("Must give at least one BCP number or 'index'; skipping.")
		return
	for ThisArg in Args:
		# Special case: 'index' returns the bcp-index.txt file
		if ThisArg == "index":
			os.system(DisplayTextCommand + os.path.join(RFCDir, "bcp-index.txt"))
			continue
		else:
			for ThisArg in Args:
				StrippedBCP = StripLeadingZeros(ThisArg)
				ThisBCPFile = os.path.join(RFCDir, "bcp", "bcp"+ StrippedBCP + ".txt")
				if os.path.exists(ThisBCPFile):
					os.system(DisplayTextCommand + ThisBCPFile)
				else:
					print("Could not find the BCP " + StrippedBCP + " as '" + ThisBCPFile + "'; skipping.")

### FillAllWGsInIETF -- Helper function for speeding up lookup of new-style charters
def FillAllWGsInIETF():
	# Get this list once to optimize if there are many WGs to look up
	try:
		os.chdir(os.path.expanduser(CharterDir))
	except:
		exit("Weird: could not chdir to " + CharterDir)
	global AllWGsInIETF
	AllWGsInIETF = {}
	for ThisCharterFile in sorted(glob.glob("charter-ietf-*")):
		CharterParts = (ThisCharterFile[13:-4]).split("-")
		# There's always a special case for the Security Area <grumble>
		if CharterParts[0:2] == ["krb", "wg"]:
			CharterParts = [ "krb-wg", CharterParts[2:] ]
		AllWGsInIETF[CharterParts[0]] = ThisCharterFile

### charter -- Open 2012-style charter files locally
def Cmd_charter(Args):
	if CheckHelp("charternew", Args): return
	if Args[0] == "":
		print("Must give at least one WG name; skipping.")
		return
	FillAllWGsInIETF()
	for ThisArg in Args:
		ThisArg = ThisArg.lower()
		MatchingWGs = fnmatch.filter(sorted(AllWGsInIETF.keys()), "*" + ThisArg + "*")
		if len(MatchingWGs) > 10:
			AllMatched = ", ".join(MatchingWGs)
			print("More than 10 WGs match '*" + ThisArg + "*' in the IETF directory. Skipping.\n" + AllMatched)
		elif len(MatchingWGs) == 0:
			print("Did not find the WG that matches '*" + ThisArg + "*' in the IETF directory.")
			print("Possibly try the 'tracker' command to see if the Datatracker has the desired data. Skipping.")
		else:
			for ThisWG in MatchingWGs:
				CharterTextFile = os.path.join(os.path.expanduser(CharterDir), AllWGsInIETF[ThisWG])
				if os.path.exists(CharterTextFile):
					os.system(DisplayTextCommand + CharterTextFile)
				else:
					print("Weird: when looking for the charter file for " + ThisWG + ", I should have found " \
						+ CharterTextFile + ", but didn't. Skipping.")

### conflict -- Show the conflict review for a draft 
def Cmd_conflict(Args):
	if CheckHelp("conflict", Args): return
	if Args[0] == "":
		print("Must give at least one draft name; skipping.")
		return
	for ThisArg in Args:
		if ThisArg.startswith("draft-"):
			# Strip any ".txt" and "-nn" from the arugment so we can match the database
			ShorterArg = re.sub(r'(\.txt)$', "", ThisArg)
			ShorterArg = re.sub(r'-\d\d$', "", ShorterArg)
			# Remove "draft-" from the beginning
			ShorterArg = ShorterArg[6:]
			WebDisplay("https://datatracker.ietf.org/doc/conflict-review-", ShorterArg)
		else:
			print("The argument to this command must begin with 'draft-'.\n")

### diff -- Show the diff between a draft and the previous one on the IETF Tools site
def Cmd_diff(Args):
	if CheckHelp("diff", Args): return
	if Args[0] == "":
		print("Must give at least one draft name; skipping.")
		return
	for ThisArg in Args:
		if ThisArg.startswith("draft-"):
			WebDisplay("https://tools.ietf.org/rfcdiff?url2=", ThisArg)
		else:
			print("The argument to this command must begin with 'draft-'.\n")

### draft -- Open drafts locally
def Cmd_draft(Args):
	if CheckHelp("draft", Args): return
	if Args[0] == "":
		print("Must give at least one draft name; skipping.")
		return
	ShowHTML = False
	ShowXML = False
	for ThisArg in Args:
		# Special case: 'abstracts" returns the 1id-abstracts.txt file
		if ThisArg == "abstracts":
			os.system(DisplayTextCommand + os.path.join(IDDir, "1id-abstracts.txt"))
			continue
		# See if we are instead supposed be showing the HTML or XML if possible
		if ThisArg == "html":
			ShowHTML = True;
			continue
		if ThisArg == "xml":
			ShowXML = True;
			continue
		# Pay attention only to expired, became-an-rfc, was replaced, and active drafts
		MatchedDraftsByStatus = { "Expired": [], "RFC": [], "Replaced": [], "Active": [] }
		# Strip any ".txt" and "-nn" from the arugment so we can match the database
		ShorterArg = re.sub(r'(\.txt)$', "", ThisArg)
		ShorterArg = re.sub(r'-\d\d$', "", ShorterArg)
		# Find all the drafts in the database that match the argument given
		for ThisDraftFromDraftsDB in IDStatusDB.keys():
			if re.search(".*" + ShorterArg + ".*", ThisDraftFromDraftsDB):
				ThisStatus = IDStatusDB[ThisDraftFromDraftsDB]["status"]
				if ThisStatus in MatchedDraftsByStatus.keys():
					MatchedDraftsByStatus[ThisStatus].append(ThisDraftFromDraftsDB)
		# Report on the drafts found for expired, became an RFC, and replaced
		if not(QuietDraft):
			if MatchedDraftsByStatus["Expired"]:
				print("Matching drafts that have expired:")
				for ThisExpired in sorted(MatchedDraftsByStatus["Expired"]):
					print("  " + ThisExpired + " (last revised " + IDStatusDB[ThisExpired]["last-revised"] + ")")
				print()
			if MatchedDraftsByStatus["RFC"]:
				print("Matching drafts that became RFCs:")
				for ThisBecameRFC in sorted(MatchedDraftsByStatus["RFC"]):
					print("  " + ThisBecameRFC + " (became RFC " + IDStatusDB[ThisBecameRFC]["became-rfc"] + ")")
				print()
			if MatchedDraftsByStatus["Replaced"]:
				print("Matching drafts that were replaced:")
				for ThisWasReplaced in sorted(MatchedDraftsByStatus["Replaced"]):
					print("  " + ThisWasReplaced + " (replaced by " + IDStatusDB[ThisWasReplaced]["replaced-by"] + ")")
				print()
		# If there are no active drafts that match this argument, say something and go to the next argument
		if not(MatchedDraftsByStatus.get("Active")):
			print("No active drafts matched the substring '" + ThisArg + "'.")
			continue
		# If there are too many matched active drafts, list them and go to the next argument
		if len(MatchedDraftsByStatus["Active"]) > MaxDrafts:
			print("There are more than " + str(MaxDrafts) + " active drafts that match the string '" \
				+ ThisArg + "'; not displaying.\nYou can raise this count with ", end="")
			if FromCommandLine:
				print(" the '--maxdrafts' command-line argument,\nsuch as '--maxdrafts=40'.")
			else:
				print(" the 'maxdrafts' command,\nsuch as 'maxdrafts 40'.")
			for ThisOverMax in MatchedDraftsByStatus["Active"]:
				print("  " + ThisOverMax)
			continue
		# Display the active drafts that match this argument
		for ThisActiveDraft in sorted(MatchedDraftsByStatus["Active"]):
			# If it is in Auth48, display it from that directory only
			ThisAuth48 = FindDraftInAuth48(ThisActiveDraft)
			if ThisAuth48 != "":
				print("This Internet-Draft is in AUTH48 state; displaying " + ThisAuth48)
				WebDisplay("file:///", os.path.join(RFCDir, "authors", ThisAuth48 + "-diff.html"))
				continue
			# Display the draft from the numbered or unnumbered mirror directory, based on their preference
			if UseDraftNumbers or ShowHTML or ShowXML:
				TargetDir = IDDir
			else:
				TargetDir = ShortIDDir
			if ShowHTML:
				TheseNumberedDrafts = glob.glob(os.path.join(TargetDir, ThisActiveDraft + "*.html"))
			elif ShowXML:
				TheseNumberedDrafts = glob.glob(os.path.join(TargetDir, ThisActiveDraft + "*.xml"))
			else:
				TheseNumberedDrafts = glob.glob(os.path.join(TargetDir, ThisActiveDraft + "*"))
			# Check for missing HTML and XML	
			if len(TheseNumberedDrafts) == 0 and ShowHTML:
				print("Could not find a an HTML version of '" + ThisActiveDraft + "' in '" + TargetDir + "'; skipping.")
			elif len(TheseNumberedDrafts) == 0 and ShowXML:
				print("Could not find a an XML version of '" + ThisActiveDraft + "' in '" + TargetDir + "'; skipping.")
			else:
				for ThisToDisplay in TheseNumberedDrafts:
					if ShowHTML:
						WebDisplay("file:///", ThisToDisplay)
					else:
						os.system(DisplayTextCommand + os.path.join(TargetDir, ThisToDisplay))
				ThisIDStatus = IDStatusDB.get(ThisActiveDraft)
				print("Draft " + ThisActiveDraft + ":\n  Status: " + ThisIDStatus["status"])
				if ThisIDStatus.get("title"):
					print("  Draft title: " + ThisIDStatus.get("title"))
				if ThisIDStatus.get("authors"):
					print("  Authors: " + ThisIDStatus.get("authors"))
				if ThisIDStatus.get("last-revised"):
					print("  Last revision: " + ThisIDStatus.get("last-revised"))
				if ThisIDStatus.get("iesg-state"):
					print("  IESG state: " + ThisIDStatus.get("iesg-state"))
				if ThisIDStatus.get("intended-level"):
					print("  Intended level: " + ThisIDStatus.get("intended-level"))
				if ThisIDStatus.get("last-call-ends"):
					print("  Last call ends: " + ThisIDStatus.get("last-call-ends"))
				if ThisIDStatus.get("became-rfc"):
					print("  Became RFC:\n" + ListRFCsAndNames(ThisIDStatus.get("became-rfc")))
				if ThisIDStatus.get("replaced-by"):
					print("  Replaced by: " + ThisIDStatus.get("replaced-by"))
				if ThisIDStatus.get("wg-name"):
					print("  WG: " + ThisIDStatus.get("wg-name"))
				if ThisIDStatus.get("area-name"):
					print("  Area: " + ThisIDStatus.get("area-name"))
				if ThisIDStatus.get("ad-name"):
					print("  Area Director: " + ThisIDStatus.get("ad-name"))
				if ThisIDStatus.get("updates"):
					print("  Updates:\n" + ListRFCsAndNames(ThisIDStatus.get("updates")))
				if ThisIDStatus.get("obsoletes"):
					print("  Obsoletes:\n" + ListRFCsAndNames(ThisIDStatus.get("obsoletes")))

### draftreport -- Print report about drafts
def Cmd_draftreport(Args):
	if CheckHelp("draftreport", Args): return
	# Get the list of drafts that update and obsolete RFCs
	DraftsThatUpdate = []
	DraftsThatObsolete = []
	RFCsBeingUpdated = {}
	RFCsBeingObsoleted = {}
	for (ThisDraftName, ThisDraftData) in sorted(IDStatusDB.items()):
		if ThisDraftData.get("updates"):
			DraftsThatUpdate.append([ThisDraftName, ThisDraftData["title"], ThisDraftData["updates"]])
			for ThisRFC in ThisDraftData["updates"]:
				if ThisRFC in RFCsBeingUpdated:
					RFCsBeingUpdated[ThisRFC].append(ThisDraftName)
				else:
					RFCsBeingUpdated[ThisRFC] = [ ThisDraftName ]
		if ThisDraftData.get("obsoletes"):
			DraftsThatObsolete.append([ThisDraftName, ThisDraftData["title"], ThisDraftData["obsoletes"]])
			for ThisRFC in ThisDraftData["obsoletes"]:
				if ThisRFC in RFCsBeingObsoleted:
					RFCsBeingObsoleted[ThisRFC].append(ThisDraftName)
				else:
					RFCsBeingObsoleted[ThisRFC] = [ ThisDraftName ]
	print("There are " + str(len(DraftsThatUpdate)) + " drafts that update others:")
	for ThisDraft in DraftsThatUpdate:
		print("  " + ThisDraft[0] + "    " + ThisDraft[1] + ":\n" + ListRFCsAndNames(ThisDraft[2]))
	print("\nThere are " + str(len(DraftsThatObsolete)) + " drafts that obsolete others:")
	for ThisDraft in DraftsThatObsolete:
		print("  " + ThisDraft[0] + "    " + ThisDraft[1] + ":\n" + ListRFCsAndNames(ThisDraft[2]))
	print("\nThere are " + str(len(RFCsBeingUpdated)) + " RFCs being updated:")
	for ThisRFC in sorted(RFCsBeingUpdated):
		print(ThisRFC + ":\n" + ListIDsAndNames(RFCsBeingUpdated[ThisRFC]))
	print("\nThere are " + str(len(RFCsBeingObsoleted)) + " RFCs being obsoleted:")
	for ThisRFC in sorted(RFCsBeingObsoleted):
		print(ThisRFC + ":\n" + ListIDsAndNames(RFCsBeingObsoleted[ThisRFC]))
	# Find all the drafts that have "Updates" or "Obsoletes" lines, but are not in the database
	try:
		os.chdir(ShortIDDir)
	except:
		exit("Weird: could not chdir to " + ShortIDDir + ". Exiting.")
	DraftsThatUpdateJustNames = []
	for ThisTuple in DraftsThatUpdate:
		DraftsThatUpdateJustNames.append(ThisTuple[0])
	FilesWithUpdatesLines = subprocess.getoutput('grep -l "^Updates:" *')
	print("\nDrafts that have Updates lines but that are not in the dataset:")
	for ThisFileName in FilesWithUpdatesLines.splitlines():
		if not ThisFileName in DraftsThatUpdateJustNames:
			print("  " + ThisFileName)
			DraftLines = open(ThisFileName, mode="rt").read().splitlines()
			for LineCount in range(10):
				if DraftLines[LineCount].strip() != "":
					print("    " + DraftLines[LineCount])
	DraftsThatObsoleteJustNames = []
	for ThisTuple in DraftsThatObsolete:
		DraftsThatObsoleteJustNames.append(ThisTuple[0])
	FilesWithObsoletesLines = subprocess.getoutput('grep -l "^Obsoletes:" *')
	print("\nDrafts that have Obsoletes lines but that are not in the dataset:")
	for ThisFileName in FilesWithObsoletesLines.splitlines():
		if not ThisFileName in DraftsThatObsoleteJustNames:
			print("  " + ThisFileName)
			DraftLines = open(ThisFileName, mode="rt").read().splitlines()
			for LineCount in range(10):
				if DraftLines[LineCount].strip() != "":
					print("    " + DraftLines[LineCount])

### iesg -- Show IESG pages on the Datatracker
def Cmd_iesg(Args):
	if CheckHelp("iesg", Args): return
	if Args[0] == "":
		print("Must give at least one of 'agenda' or 'docs' as an argument; skipping.")
		return
	for ThisArg in Args:
		# If it is just a number, check for the RFC
		if ThisArg.lower() == "agenda":
			WebDisplay("https://datatracker.ietf.org/iesg/agenda", "")
		if ThisArg.lower() == "docs":
			WebDisplay("https://datatracker.ietf.org/iesg/agenda/documents", "")

### mirror -- Update the local mirror
def Cmd_mirror(Args):
	if CheckHelp("mirror", Args): return
	# See if the main directory exists; if not, try to create it
	if os.path.exists(os.path.expanduser(MirrorDir)) == False:
		try:
			os.mkdir(os.path.expanduser(MirrorDir))
		except:
			exit("The mirror directory '" + MirrorDir + "' does not exist, and could not be created. Exiting.")
	if os.path.exists(os.path.expanduser(IDDir)) == False:
		print("This appears to be the first time you are running this; it may take a long")
		print("  time. Each mirror section will be named, but the files being mirrored will")
		print("  only appear when the full directory has been mirrored; this can take hours,")
		print("  depending on network speed. You can check the progress by looking in the")
		print("  created directories.")
	# Set up the log file
	LogFile = os.path.expanduser(MirrorDir + "/mirror-log.txt")
	try:
		logf = open(LogFile, "a")
	except:
		exit("Could not open " + LogFile + " for appending. Exiting.\n")
	# Print out to both the console and log file
	def PrintLog(String):
		print(String)
		print(String, file=logf)
	PrintLog("\nMirror began at " + time.strftime("%Y-%m-%d %H:%M:%S") + "\n")
	# AllActions is the set of actions to be performed
	AllActions = [
		[ "Internet Drafts", "rsync -avz --exclude='*.pdf' --exclude='*.p7s' " +
			" --exclude='*.ps' --delete-after  rsync.ietf.org::internet-drafts " + IDDir ],
		[ "IANA", "rsync -avz --delete-after  rsync.iana.org::assignments " + IANADir ],
		[ "IESG", "rsync -avz --delete-after  rsync.ietf.org::iesg-minutes/ " + IESGDir ],
		[ "IETF", "rsync -avz --delete-after  --exclude='ipr/' " +
			"ietf.org::everything-ftp/ietf/ " + IETFDir ],
		[ "charters", "rsync -avz --delete-after  rsync.ietf.org::everything-ftp/charter/ " + CharterDir ],
		[ "conflict reviews", "rsync -avz --delete-after  rsync.ietf.org::everything-ftp/conflict-reviews/ " + ConflictDir ],
		[ "status changes", "rsync -avz --delete-after  rsync.ietf.org::everything-ftp/status-changes/ " + StatusDir ],
		[ "RFCs", "rsync -avz --delete-after " +
			" --exclude='tar*' --exclude='search*' --exclude='PDF-RFC*' " +
			" --exclude='tst/' --exclude='pdfrfc/' --exclude='internet-drafts/' " +
			" --exclude='ien/' rfc-editor.org::everything-ftp/in-notes/ " + RFCDir ]
		]
	for DoThis in AllActions:
		PrintLog("Starting " + DoThis[0])
		OutLines = []
		p = subprocess.Popen(DoThis[1], bufsize=-1, shell=True, stdout=subprocess.PIPE)
		while p.poll() is None:
			OutLines.append(p.stdout.readline())
		TheOut = ""
		for ThisLine in OutLines:
			# Need the following to prevent printing and parsing problems later
			ThisLine = ThisLine.decode("ascii")
			if ThisLine.startswith("receiving "): continue
			if ThisLine.startswith("sent "): continue
			if ThisLine.startswith("total "): continue
			if ThisLine.startswith("skipping non-regular file "): continue
			if ThisLine.endswith('.listing" [1]\n'): continue
			if ThisLine == "\n": continue
			TheOut += ThisLine
		PrintLog(TheOut)

	# Do the filling of the short-name directory
	PrintLog("Filling short-name directory")
	# See if the directory mirrorded from the IETF exists and get the list of drafts
	if os.path.exists(IDDir) == False:
		exit("The directory with the drafts, " + IDDir + ", does not exist. Exiting.")
	elif os.path.isdir(IDDir) == False:
		exit(IDDir + "is not a directory. Exiting.")
	try:
		os.chdir(IDDir)
	except:
		exit("Weird: could not chdir to " + IDDir + ". Exiting.")
	# Note that this is only making short names for .txt files, not any of the others
	TheIDs = sorted(glob.glob("draft-*.txt"))
	# See if the directory to be copied to exists; if so, delete all the files there
	if os.path.exists(ShortIDDir) == False:
		try:
			os.mkdir(ShortIDDir)
		except:
			exit("The directory where the shorter-named drafts will go, " + ShortIDDir + ", could not be created. Exiting.")
	elif os.path.isdir(ShortIDDir) == False:
		exit(ShortIDDir + "is not a directory. Exiting.")
	try:
		os.chdir(ShortIDDir)
	except:
		exit("Weird: could not chdir to " + ShortIDDir + ". Exiting.")
	for ToDel in glob.glob("*"):
		if os.path.isdir(ToDel):
			exit("Found a directory in " + ShortIDDir + ". Exiting.")
		os.unlink(ToDel)
	# Determine the shorter name and link the file with the destination
	for ThisDraftName in TheIDs:
		# Strip off "-nn.txt"
		ShorterName = ThisDraftName[:-7]
		# Test if the shorter name already exists; if so, nuke it
		#   This is based on the the assumption that there are two drafts where the version numbers
		#   are different, and because this is sorted, the higher ones should come later.
		if os.path.exists(os.path.join(ShortIDDir, ShorterName)):
			os.unlink(os.path.join(ShortIDDir, ShorterName))
		try:
			os.link(os.path.join(IDDir, ThisDraftName), os.path.join(ShortIDDir, ShorterName))
		except OSError as e:
			print("For '" + ThisDraftName + "', got error: " + str(e) + ". Skipping.")	

	PrintLog("Making the status databases")
	# RFC status
	TagBase = "{http://www.rfc-editor.org/rfc-index}"
	try:
		ParsedRFCDB = ElementTree.parse(os.path.join(RFCDir, "rfc-index.xml"))
	except:
		exit("Weird: could not find '" + os.path.join(RFCDir, "rfc-index.xml") + "' when building the status index. Exiting.")
	TreeRoot = ParsedRFCDB.getroot()
	RFCStatus = {}
	LookForFields = ("obsoleted-by", "updated-by", "obsoletes", "updates", "is-also")
	for ThisTopNode in TreeRoot:
		if ThisTopNode.tag == TagBase + "rfc-entry":
			ThisRFCNum = StripLeadingZeros(ThisTopNode.find(TagBase + "doc-id").text.replace("RFC", ""))
			RFCStatus[ThisRFCNum] = {}
			for ThisLookedFor in LookForFields:
				if ThisTopNode.findall(TagBase + ThisLookedFor):
					RFCStatus[ThisRFCNum][ThisLookedFor] = []
					for ThisFoundOuterElement in ThisTopNode.findall(TagBase + ThisLookedFor):
						for ThisFoundInnerElement in ThisFoundOuterElement.findall(TagBase + "doc-id"):
							RFCStatus[ThisRFCNum][ThisLookedFor].append(StripLeadingZeros(ThisFoundInnerElement.text.replace("RFC", "")))
			if ThisTopNode.findall(TagBase + "errata-url"):
				RFCStatus[ThisRFCNum]["errata"] = True
			ThisTitle = ThisTopNode.find(TagBase + "title").text
			if ThisTitle:
				RFCStatus[ThisRFCNum]["title"] = ThisTitle
			CurrStat = ThisTopNode.find(TagBase + "current-status").text
			if (CurrStat and CurrStat != "UNKNOWN"):
				RFCStatus[ThisRFCNum]["current-status"] = CurrStat
			RFCStatus[ThisRFCNum]["authors"] = []
			for ThisFoundOuterAuthor in ThisTopNode.findall(TagBase + "author"):
				for ThisFoundInnerAuthor in ThisFoundOuterAuthor.findall(TagBase + "name"):
					RFCStatus[ThisRFCNum]["authors"].append(ThisFoundInnerAuthor.text)
	# BCP Status
	BCPStatus = {}
	for ThisTopNode in TreeRoot:
		if ThisTopNode.tag == TagBase + "bcp-entry":
			ThisBCPNum = StripLeadingZeros(ThisTopNode.find(TagBase + "doc-id").text.replace("BCP", ""))
			BCPStatus[ThisBCPNum] = {}
			BCPStatus[ThisBCPNum]["is-also"] = []
			for ThisFoundOuterAlso in ThisTopNode.findall(TagBase + "is-also"):
				for ThisFoundInnerAlso in ThisFoundOuterAlso.findall(TagBase + "doc-id"):
					BCPStatus[ThisBCPNum]["is-also"].append(StripLeadingZeros(ThisFoundInnerAlso.text.replace("RFC", "")))
	# STD Status
	STDStatus = {}
	for ThisTopNode in TreeRoot:
		if ThisTopNode.tag == TagBase + "std-entry":
			ThisSTDNum = StripLeadingZeros(ThisTopNode.find(TagBase + "doc-id").text.replace("STD", ""))
			STDStatus[ThisSTDNum] = {}
			STDStatus[ThisSTDNum]["is-also"] = []
			for ThisFoundOuterAlso in ThisTopNode.findall(TagBase + "is-also"):
				for ThisFoundInnerAlso in ThisFoundOuterAlso.findall(TagBase + "doc-id"):
					STDStatus[ThisSTDNum]["is-also"].append(StripLeadingZeros(ThisFoundInnerAlso.text.replace("RFC", "")))
			ThisTitle = ThisTopNode.find(TagBase + "title").text
			if ThisTitle:
				STDStatus[ThisSTDNum]["title"] = ThisTitle

	# Draft status
	try:
		AllIDStatusLines = open(IDDir + "/all_id2.txt", mode="r").readlines()
	except:
		exit("Weird: could not read all_id2.txt to make the I-D status database. Exiting.")
	IDStatus = {}
	for ThisLine in AllIDStatusLines:
		if ThisLine.strip() == "": continue  # Skip accidental blank lines
		if ThisLine[0] == "#": continue  # Skip comment lines
		TheFields = ThisLine.split("\t")
		# The key is the draft name minus the "-nn"
		IDStatus[TheFields[0][0:-3]] = { \
			"status": TheFields[2], \
			"iesg-state": TheFields[3], \
			"became-rfc": TheFields[4], \
			"replaced-by": TheFields[5], \
			"last-revised": TheFields[6], \
			"wg-name": TheFields[7], \
			"area-name": TheFields[8], \
			"ad-name": TheFields[9], \
			"intended-level": TheFields[10], \
			"last-call-ends": TheFields[11], \
			"file-types": TheFields[12], \
			"title": TheFields[13], \
			"authors": TheFields[14].rstrip() }
	# Look for draft updating and obsoleting status by going through all the files in the short directory
	try:
		os.chdir(ShortIDDir)
	except:
		exit("Weird: could not chdir to " + ShortIDDir + ". Exiting.")
	for ThisShort in sorted(glob.glob("*")):
		# all_id2.txt is only updated periodically (in early 2021, once a day)
		#   So a new draft on disk might not be there. If that happens, skip over it.
		if not ThisShort in IDStatus:
			continue
		# Do the updates processing first
		AllDraftLines = open(ThisShort, mode="rt").read().splitlines()
		UpdatesStartsOn = 0
		for (LineNum, ThisLine) in enumerate(AllDraftLines[:20]):
			if ThisLine.startswith("Updates: "):
				UpdatesStartsOn = LineNum
				break
		if UpdatesStartsOn > 0:
			FiveLines = AllDraftLines[UpdatesStartsOn:UpdatesStartsOn+5]
			# Remove "Updates: " from first line
			FiveLines[0] = FiveLines[0].replace("Updates: ", "")
			# Process the lines
			for LineNumber in range(len(FiveLines)):
				# Get rid of spaces at the start and end the line
				FiveLines[LineNumber] = FiveLines[LineNumber].strip()
				# Stop at multiple spaces
				if "   " in FiveLines[LineNumber]:
					FiveLines[LineNumber] = FiveLines[LineNumber][:FiveLines[LineNumber].index("   ")]
				# Get rid of extraneous text on the lines
				FiveLines[LineNumber] = FiveLines[LineNumber].replace("(if approved)", "").replace("(if", "").replace("approved)", "")
				FiveLines[LineNumber] = FiveLines[LineNumber].replace("RFC", "").replace("rfc", "").replace("none", "")
				# Strip off spaces at the beginning and end that are left
				FiveLines[LineNumber] = FiveLines[LineNumber].strip()
				# Some drafts (erroneously) say that they update other drafts
				#   Look for a "-" in a draft name; if it appears, get rid of the whole line
				if "-" in FiveLines[LineNumber]:
					FiveLines[LineNumber] = ""
					continue
				# If the processed line now does not start with a three-digit number, get rid of the whole line
				if not re.match(r"\d\d\d", FiveLines[LineNumber]):
					FiveLines[LineNumber] = ""
					continue
			# Put all the lines together, and parse out the RFC numbers
			if not FiveLines == ["", "", "", "", ""]:
				UpdatesListText = " ".join(FiveLines)
				UpdatesListFull = UpdatesListText.split(" ")
				UpdatesList = []
				# Get rid of the blanks, find items that have a comma but no spaces, and remove trailing commas
				for ThisItem in UpdatesListFull:
					if ThisItem == "":
						continue
					elif ThisItem.endswith(","):
						UpdatesList.append(ThisItem[:-1])
					elif "," in ThisItem:
						UpdatesList.extend(ThisItem.split(","))
					else:
						UpdatesList.append(ThisItem)
				try:
					IDStatus[ThisShort]["updates"] = UpdatesList
				except Exception as e:
					print("Did not find " + ThisShort + " in status list, so skipping.")
		# Do the obsoletes processing second
		ObsoletesStartsOn = 0
		for (LineNum, ThisLine) in enumerate(AllDraftLines[:20]):
			if ThisLine.startswith("Obsoletes: "):
				ObsoletesStartsOn = LineNum
				break
		if ObsoletesStartsOn > 0:
			FiveLines = AllDraftLines[ObsoletesStartsOn:ObsoletesStartsOn+5]
			# Remove "Obsoletes: " from first line
			FiveLines[0] = FiveLines[0].replace("Obsoletes: ", "")
			# Process the lines
			for LineNumber in range(len(FiveLines)):
				# Get rid of spaces at the start and end the line
				FiveLines[LineNumber] = FiveLines[LineNumber].strip()
				# Stop at multiple spaces
				if "   " in FiveLines[LineNumber]:
					FiveLines[LineNumber] = FiveLines[LineNumber][:FiveLines[LineNumber].index("   ")]
				# Get rid of extraneous text on the lines
				FiveLines[LineNumber] = FiveLines[LineNumber].replace("(if approved)", "").replace("(if", "").replace("approved)", "")
				FiveLines[LineNumber] = FiveLines[LineNumber].replace("RFC", "").replace("rfc", "").replace("none", "")
				# Some drafts (erroneously) say that they update other drafts
				#   Look for a "-" in a draft name; if it appears, get rid of the whole line
				if "-" in FiveLines[LineNumber]:
					FiveLines[LineNumber] = ""
					continue
				# If the processed line now does not start with a three-digit number, get rid of the whole line
				if not re.match(r"\d\d\d", FiveLines[LineNumber]):
					FiveLines[LineNumber] = ""
					continue
			# Put all the lines together, and parse out the RFC numbers
			if not FiveLines == ["", "", "", "", ""]:
				ObsoletesListText = " ".join(FiveLines)
				ObsoletesListFull = ObsoletesListText.split(" ")
				ObsoletesList = []
				# Get rid of the blanks, find items that have a comma but no spaces, and remove trailing commas
				for ThisItem in ObsoletesListFull:
					if ThisItem == "":
						continue
					elif ThisItem.endswith(","):
						ObsoletesList.append(ThisItem[:-1])
					elif "," in ThisItem:
						ObsoletesList.extend(ThisItem.split(","))
					else:
						ObsoletesList.append(ThisItem)
				IDStatus[ThisShort]["obsoletes"] = ObsoletesList
	# Dump the JSON
	ToDump = [
		[RFCStatusFileLoc, RFCStatus],
		[IDStatusFileLoc, IDStatus],
		[BCPStatusFileLoc, BCPStatus],
		[STDStatusFileLoc, STDStatus]
	]
	for ThisDump in ToDump:
		try:
			with open(ThisDump[0], mode="w") as statusf:
				json.dump(ThisDump[1], statusf, indent=1)
		except:
			exit("Could not dump status info to '" + RFCStatusFileLoc + "'. Exiting.")	
	# Finish up
	PrintLog("\nMirror ended at " + time.strftime("%Y-%m-%d %H:%M:%S"))
	logf.close()

### rfc -- Open RFCs locally
def Cmd_rfc(Args):
	if CheckHelp("rfc", Args): return
	if Args[0] == "":
		print("Must give at least one RFC name or number; skipping.")
		return
	for ThisArg in Args:
		# Special case: 'index' returns the rfc-index.txt file
		if ThisArg == "index":
			os.system(DisplayTextCommand + os.path.join(RFCDir, "rfc-index.txt"))
			continue
		# Look for different ways they may have specified it
		RFCTests = [ ThisArg, ThisArg + ".txt", "rfc" + ThisArg, "rfc" + ThisArg + ".txt" ]
		FoundRFC = False
		for ThisTest in RFCTests:
			# Also check in the AUTH48 directory
			for WhichDir in (RFCDir, RFCDir + "/authors"):
				if os.path.exists(os.path.join(WhichDir, ThisTest)):
					FoundRFC = True
					os.system(DisplayTextCommand + os.path.join(WhichDir, ThisTest))
					break
		if FoundRFC == False:
			print("Could not find an RFC for '" + ThisArg + "' in '" + RFCDir + "'; skipping.")
		# Get the status of the RFC
		ShorterArg = re.sub("^rfc", "", ThisArg)
		ShorterArg = re.sub(".txt%", "", ShorterArg)
		ThisRFCStatus = RFCStatusDB.get(ShorterArg)
		# Be sure the status exists
		if ThisRFCStatus:
			print("RFC " + ShorterArg + ":")
			if ThisRFCStatus.get("is-also"):
				print("  Is also:\n" + ListRFCsAndNames(ThisRFCStatus.get('is-also')))
			if ThisRFCStatus.get("obsoleted-by"):
				print("  Obsoleted by:\n" + ListRFCsAndNames(ThisRFCStatus.get('obsoleted-by')))
			if ThisRFCStatus.get("obsoletes"):
				print("  Obsoletes:\n" + ListRFCsAndNames(ThisRFCStatus.get('obsoletes')))
			if ThisRFCStatus.get("updated-by"):
				print("  Updated by:\n" + ListRFCsAndNames(ThisRFCStatus.get('updated-by')))
			if ThisRFCStatus.get("updates"):
				print("  Updates:\n" + ListRFCsAndNames(ThisRFCStatus.get('updates')))
			if ThisRFCStatus.get("errata") == True:
				print("  Has errata")
		else:
			print("Weird: did not find status in the database for RFC " + ThisArg + "; skipping.")
		HasUpdates = []
		HasObsoletes = []
		for ThisDraft in sorted(IDStatusDB.keys()):
			if IDStatusDB[ThisDraft].get("updates") and ShorterArg in IDStatusDB[ThisDraft]["updates"]:
				HasUpdates.append(ThisDraft)
			if IDStatusDB[ThisDraft].get("obsoletes") and ShorterArg in IDStatusDB[ThisDraft]["obsoletes"]:
				HasObsoletes.append(ThisDraft)
		if HasUpdates:
			print("  Is possibly being updated by\n" + ListIDsAndNames(HasUpdates))
		if HasObsoletes:
			print("  Is possibly being obsoleted by\n" + ListIDsAndNames(HasObsoletes))
		print()

### rfcextra -- Open RFCs locally and also open related RFCs (updates, obsoleted, errata...)
def Cmd_rfcextra(Args):
	if CheckHelp("rfcextra", Args): return
	if Args[0] == "":
		print("Must give at least one RFC name or number; skipping.")
		return
	for ThisArg in Args:
		# First try to open the RFC itself
		Cmd_rfc([ThisArg])
		# Then get the status of the RFC and open RFCs and errata that happened later
		ThisRFCStatus = RFCStatusDB.get(ThisArg)
		# If the status exists for this RFC, display additional information and open what was found
		if ThisRFCStatus:
			if ThisRFCStatus.get("obsoleted-by"):
				for ThisObsoleted in ThisRFCStatus.get("obsoleted-by"):
					print("RFC " + ThisArg + " was obsoleted by RFC " + ThisObsoleted)
					Cmd_rfc([ThisObsoleted])
			if ThisRFCStatus.get("updated-by"):
				for ThisUpdated in ThisRFCStatus.get("updated-by"):
					print("RFC " + ThisArg + " was updated by RFC " + ThisUpdated)
					Cmd_rfc([ThisUpdated])
			if ThisRFCStatus.get("errata") == True:
				print("RFC " + ThisArg + " has errata")
				WebDisplay("https://www.rfc-editor.org/errata_search.php?rfc=", ThisArg)

### rfcinfo -- Show RFC information on the RFC Editor site
def Cmd_rfcinfo(Args):
	if CheckHelp("rfcinfo", Args): return
	if Args[0] == "":
		print("Must give at least one RFC number; skipping.")
		return
	for ThisArg in Args:
		# If it is just a number, check for the RFC
		if ThisArg.isdigit():
			WebDisplay("https://www.rfc-editor.org/info/rfc", ThisArg)
		# If it starts with "rfc" and rest are digits, it is also an RFC
		elif (ThisArg.startswith("rfc") and  ThisArg[3:].isdigit()):
			WebDisplay("https://www.rfc-editor.org/info/", ThisArg)
		else:
			print("This command is for finding RFCs on the RFC Editor's site web site.\n")

### rg -- Show IRTF RGs on the Datatracker
def Cmd_rg(Args):
	if CheckHelp("rg", Args): return
	if Args[0] == "":
		print("Must give at least one RG name; skipping.")
		return
	for ThisArg in Args:
		ThisArg = ThisArg.lower()
		WebDisplay("https://datatracker.ietf.org/rg/", ThisArg)

### std -- Open THe RFCs in a STD locally
def Cmd_std(Args):
	if CheckHelp("std", Args): return
	if Args[0] == "":
		print("Must give at least one STD number or 'index'; skipping.")
		return
	for ThisArg in Args:
		StrippedSTD = StripLeadingZeros(ThisArg)
		# Print out the relevant material from the STDStatusDB
		ThisStatus = STDStatusDB.get(StrippedSTD)
		if ThisStatus:
			if STDStatusDB[StrippedSTD].get("is-also"):
				print("RFCs in STD:" + StrippedSTD)
				for ThisRFC in STDStatusDB[StrippedSTD]["is-also"]:
					print(ListRFCsAndNames([ThisRFC]))
					FullPathToRFC = (os.path.join(RFCDir, "rfc" + ThisRFC + ".txt"))
					if os.path.exists(FullPathToRFC):
						os.system(DisplayTextCommand + FullPathToRFC)
					else:
						print("Could not find " + FullPathToRFC)
			else:
				print("The database says that there are no RFCs in STD" + StrippedSTD)
		else:
			print("Did not find any status information in the STD status database")

### stdonly -- Open STD documents locally
def Cmd_stdonly(Args):
	if CheckHelp("stdonly", Args): return
	if Args[0] == "":
		print("Must give at least one STD number or 'index'; skipping.")
		return
	for ThisArg in Args:
		# Special case: 'index' returns the std-index.txt file
		if ThisArg == "index":
			os.system(DisplayTextCommand + os.path.join(RFCDir, "std-index.txt"))
			continue
		else:
			for ThisArg in Args:
				StrippedSTD = StripLeadingZeros(ThisArg)
				ThisSTDFile = os.path.join(RFCDir, "std", "std"+ StrippedSTD + ".txt")
				if os.path.exists(ThisSTDFile):
					os.system(DisplayTextCommand + ThisSTDFile)
				else:
					print("Could not find the STD " + StrippedSTD + " as '" + ThisSTDFile + "'; skipping.")

### tools -- Show RFCs, WGs, and drafts on the IETF Tools site
def Cmd_tools(Args):
	if CheckHelp("tools", Args): return
	if Args[0] == "":
		print("Must give at least one RFC, WG, or draft name; skipping.")
		return
	for ThisArg in Args:
		ThisArg = ThisArg.lower()
		# If it is just a number, check for the RFC
		if ThisArg.isdigit():
			WebDisplay("https://tools.ietf.org/html/rfc", ThisArg)
		# If it starts with "rfc" and rest are digits, it is also an RFC
		elif (ThisArg.startswith("rfc") and  ThisArg[3:].isdigit()):
			WebDisplay("https:tools.ietf.org/html/", ThisArg)
		# If it isn't an RFC and it has no hyphens, assume it is a WG
		elif ThisArg.find("-") == -1:
			WebDisplay("https:tools.ietf.org/wg/", ThisArg)
		# Otherwise, assume it is a draft; this might get a 404
		elif ThisArg.startswith("draft-"):
			WebDisplay("https:tools.ietf.org/html/", ThisArg)
		else:
			print("This command is for finding RFCs, WGs (with no hypens) or drafts\n(that start with 'draft-')" \
				+ " on the IETF Tools web site.\n")

### tracker -- Show WGs and draft statuses on the Datatracker
def Cmd_tracker(Args):
	if CheckHelp("tracker", Args): return
	if Args[0] == "":
		print("Must give at least one WG or draft name; skipping.")
		return
	for ThisArg in Args:
		ThisArg = ThisArg.lower()
		# If it is just a number, check for the RFC
		if ThisArg.isdigit():
			WebDisplay("https://datatracker.ietf.org/doc/rfc", ThisArg)
		# If it starts with "rfc" and rest are digits, it is also an RFC
		elif (ThisArg.startswith("rfc") and  ThisArg[3:].isdigit()):
			WebDisplay("https://datatracker.ietf.org/doc/", ThisArg)
		# If it isn't an RFC and it has no hyphens, assume it is a WG
		elif ThisArg.find("-") == -1:
			WebDisplay("https://datatracker.ietf.org/wg/", ThisArg)
		# If not, assume it is a draft
		elif ThisArg.startswith("draft-"):  # This might get a 404
			WebDisplay("https://datatracker.ietf.org/doc/", ThisArg)
		else:
			print("This command is for finding WGs (with no hypens) or drafts (that start with 'draft-')" \
				+ " on the IETF Datatracker.\n")

# For showing help when --help or -h is given on the command line
def ShowCommandLineHelp(ignore1, ignore2, ignore3, ignore4):
	CheckHelp("allshellcmds", "__helptext__")
	exit()

# The real program starts here
if __name__ == "__main__":
	Parse = optparse.OptionParser(add_help_option=False, usage="Something here")
	# Don't display tombstones unless option is given
	Parse.add_option("--tombstones", action="store_true", dest="DisplayTombstones", default=False)
	# Maximum number of drafts to display
	Parse.add_option("--maxdrafts", action="store", type="int", dest="MaxDrafts", default=10)
	# Only open drafts from directory with full draft names (including version numbers)
	Parse.add_option("--usedraftnumbers", action="store_true", dest="UseDraftNumbers", default=False)
	# Normally have the "draft" and "rfc" commands be verbose
	Parse.add_option("--quiet", action="store_true", dest="QuietDraft", default=False)
	# Set up the help
	Parse.add_option("--help", "-h", action="callback", callback=ShowCommandLineHelp)
	(Opts, RestOfArgs) = Parse.parse_args()
	# Define these top-level variables to make it easier to change them from the config file
	DisplayTombstones = Opts.DisplayTombstones
	MaxDrafts = Opts.MaxDrafts
	UseDraftNumbers = Opts.UseDraftNumbers
	QuietDraft = Opts.QuietDraft

	ConfigFile = ""
	for ThisPlace in ConfigPlaces:
		if os.path.exists(os.path.expanduser(ThisPlace)):
			ConfigFile = ThisPlace
			break
	if ConfigFile == "":
		exit("Could not find a configuration file in " + " or ".join(ConfigPlaces) + "\nExiting.")

	# Initial empty string values for the configuration information
	MirrorDir = IDDir = ShortIDDir = IANADir = IESGDir = IETFDir = CharterDir = ""
	ConflictDir = StatusDir = RFCDir = DisplayTextCommand = DisplayWebCommand = ""
	# Get the variable names for the directories and display mechanisms
	try:
		Configs = open(os.path.expanduser(ConfigFile), mode="r").read()
	except:
		exit("Could not open '" + os.path.expanduser(ConfigFile) + "' for input. Exiting.")
	try:
		exec(Configs)
	except Exception as this_e:
		exit("Failed during exec of " + ConfigFile + ": '" + this_e + "'. Exiting.")

	# Location of the JSON files (which could not be complete until we got the config)
	IDStatusFileLoc = os.path.join(os.path.expanduser(IDDir), "ietf-id-status.json")
	RFCStatusFileLoc = os.path.join(os.path.expanduser(RFCDir), "ietf-rfc-status.json")
	BCPStatusFileLoc = os.path.join(os.path.expanduser(RFCDir), "ietf-bcp-status.json")
	STDStatusFileLoc = os.path.join(os.path.expanduser(RFCDir), "ietf-std-status.json")

	# All the variables from the config file must be defined, and the named directories must exist.
	#   This relies on the rarely-used method of finding variable names in globals() and changing them there
	TheDirectoryNamesAsStrings = [ "MirrorDir", "IDDir", "ShortIDDir", "IANADir", "IESGDir", "IETFDir", "RFCDir" ]
	for ThisDirName in TheDirectoryNamesAsStrings:
		# dir() returns the list of names currently defined; use this to be sure that the config file defined everything
		if globals()[ThisDirName] == "":
			exit("The variable '" + ThisDirName + "' was not defined in " + ConfigFile + ". Exiting.")
		# Expand the user in any of the names
		globals()[ThisDirName] = os.path.expanduser(globals()[ThisDirName])
		if not(os.path.exists(globals()[ThisDirName])):
			try:
				os.mkdir(globals()[ThisDirName])
			except:
				exit("Attempting to create " + globals()[ThisDirName] + " failed. Exiting.")
	# If the JSON databases don't exit yet, create empty files for them
	for ThisJSON in (IDStatusFileLoc, RFCStatusFileLoc, BCPStatusFileLoc, STDStatusFileLoc):
		if not os.path.exists(ThisJSON):
			ThisF = open(ThisJSON, mode="wt")
			ThisF.write("{}\n")
			ThisF.close()
	# The display mechanisms can be blank
	# Set defaults for the desplay commands if they are not set
	if DisplayTextCommand == "":
		# If DisplayTextCommand is not set but the EDITOR environment variable is, use EDITOR instead
		if os.environ.get("EDITOR", "") != "":
			DisplayTextCommand = os.environ["EDITOR"] + " "
		else:
			DisplayTextCommand = "less "
	if DisplayWebCommand == "":
		DisplayWebCommand = "less "  # This is a terrible fallback, of course

	# Open the status databases
	try:
		with open(RFCStatusFileLoc, mode="r") as statusf:
			RFCStatusDB = json.load(statusf)
	except:
		exit("Weird: could not get data from the RFC status database, '" + RFCStatusFileLoc + "'. Exiting.")
	try:
		with open(IDStatusFileLoc, mode="r") as statusf:
			IDStatusDB = json.load(statusf)
	except:
		exit("Weird: could not get data from the ID status database, '" + IDStatusFileLoc + "'. Exiting.")
	try:
		with open(BCPStatusFileLoc, mode="r") as statusf:
			BCPStatusDB = json.load(statusf)
	except:
		exit("Weird: could not get data from the ID status database, '" + BCPStatusFileLoc + "'. Exiting.")
	try:
		with open(STDStatusFileLoc, mode="r") as statusf:
			STDStatusDB = json.load(statusf)
	except:
		exit("Weird: could not get data from the ID status database, '" + STDStatusFileLoc + "'. Exiting.")


	# The "ietf" command can be called with no arguments to go to the internal command processor
	#    It is often called as "ietf" with arguments from the KnownCommand list.
	if RestOfArgs == []:
		FromCommandLine = False
		try:
			OurCLI().cmdloop()
		except KeyboardInterrupt:
			exit("\n^C caught. Exiting.")
	else:
		FromCommandLine = True
		GivenCmd = RestOfArgs[0]
		if GivenCmd in KnownCmds:
			globals()["Cmd_" + GivenCmd](RestOfArgs[1:])
		else:
			exit("Found a bad command: " + GivenCmd + ". Exiting.")
