Perform a simultaneous search of multiple log files using a Python script

Summary

When attempting to diagnose and troubleshoot errors, log files may contain detailed information of the problems encountered. However, some applications can generate multiple log files which may amount to a dozen or hundred files. For example, ArcGIS Server or ArcSDE may generate several server debug logs or SDE intercept log files. Searching and reviewing each log to identify a specific keyword related to the issues can be time consuming.

A simultaneous search of multiple log files for specific keywords can be performed using a Python script.

Procedure

The following is a sample python script separated into several sections for the purpose of this article. Combine all sections and execute the script:

1. Import the necessary libraries.

   import os

2. Specify the necessary parameters for the folder location, search keywords, and case sensitivity of the keywords.

   folder = r"E:\temp\logs"
   
   search_term = r"Keywords"
   
   case_sensitive = False

   Note:
   The case_sensitive parameter requires a boolean value, either True or False.

3. Define a new list index and the necessary array parameters for the search_files function.

   def search_files(folder, search_term, case_sensitive=False, results_file=""):
   fileslist = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
       foundinlist = []
       linelist = []
       fileerrors = 0
       totallines = 0
       for file in fileslist:
           print("Searching " + file)
           try:
               with open(os.path.join(folder, file)) as text:
                   if case_sensitive:
                       if search_term in text.read():
                           linenum = 1
                           foundinlist.append(file)
                           lineresults = "-- " + file + ":\n"
                           text.seek(0)
                           for line in text:
                               if search_term in line:
                                   lineresults = lineresults + str(linenum) + ", "
                                   totallines += 1
                               linenum += 1
                           linelist.append(lineresults[:-2])
   
                   else:
                       if search_term.lower() in text.read().lower():
                           linenum = 1
                           foundinlist.append(file)
                           lineresults = "-- " + file + ":\n"
                           text.seek(0)
                           for line in text:
                               if search_term.lower() in line.lower():
                                   lineresults = lineresults + str(linenum) + ", "
                                   totallines += 1
                               linenum += 1
                           linelist.append(lineresults[:-2])
           except Exception as e:
               print("Error reading file: " + str(e))
               fileerrors += 1

4. The print function prompts the system to return a statement if the search is successful or not.

   print(("Done searching " + str(len(fileslist) - fileerrors) + " of " + str(len(fileslist))
              + " files for '" + search_term + "'"))
       if fileerrors > 0:
           print("--> Error opening " + str(fileerrors) + " file(s)")

5. This len function creates the list of results with full details.

   # results
       if len(foundinlist) > 0:
           print("\n\nSearch term found in " + str(len(foundinlist)) + " file(s):")
           for i in foundinlist:
               print(i)
           print("\n\nSearch term found on " + str(totallines) + " line(s):")
           for i in linelist:
               print(i)
   
           if results_file != "":
               results = ((["Searched " + str(len(fileslist) - fileerrors) + " of " + str(len(fileslist))
                            + " files for '" + search_term + "'"])
                          + ["\n\nSearch term found in " + str(len(foundinlist)) + " file(s):\n"]
                          + [i + "\n" for i in foundinlist]
                          + ["\n\nSearch term found on " + str(totallines) + " line(s)\n"]
                          + [i + "\n" for i in linelist]
                          )
               try:
                   with open(results_file, "w") as re_file:
                       re_file.writelines(results)
                       print("\nResults file written to:\n" + results_file)
               except Exception as e:
                   print("\nResults file could NOT be written! Check your results file path")
                   print(e)
           return True
   
       else:
           print("\nNo matches found")
           if results_file != "":
               print("\nResults file NOT written (no matches)")
           return False

6. The search_files function calls and executes all the functions and defined parameters listed above.

   search_files(folder, search_term, case_sensitive, results_file)

The following is the combined sample script:

################################################################################
################################################################################

# Edit search parameters then execute the script.
# The function will attempt to open and search all files in the folder.
# Folders and unreadable files will be skipped.

# Folder path where files to search are located
folder = r"E:\temp\logs"

# Search term
search_term = r"search term"

# Case sensitivity, True or False
case_sensitive = False

# Path to write the results to file if desired (e.g. 'E:\temp\results.txt'),
#  "" for no results file.
results_file = r""

################################################################################
################################################################################


import os

def search_files(folder, search_term, case_sensitive=False, results_file=""):
    fileslist = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
    foundinlist = []
    linelist = []
    fileerrors = 0
    totallines = 0
    for file in fileslist:
        print("Searching " + file)
        try:
            with open(os.path.join(folder, file)) as text:
                if case_sensitive:
                    if search_term in text.read():
                        linenum = 1
                        foundinlist.append(file)
                        lineresults = "-- " + file + ":\n"
                        text.seek(0)
                        for line in text:
                            if search_term in line:
                                lineresults = lineresults + str(linenum) + ", "
                                totallines += 1
                            linenum += 1
                        linelist.append(lineresults[:-2])

                else:
                    if search_term.lower() in text.read().lower():
                        linenum = 1
                        foundinlist.append(file)
                        lineresults = "-- " + file + ":\n"
                        text.seek(0)
                        for line in text:
                            if search_term.lower() in line.lower():
                                lineresults = lineresults + str(linenum) + ", "
                                totallines += 1
                            linenum += 1
                        linelist.append(lineresults[:-2])
        except Exception as e:
            print("Error reading file: " + str(e))
            fileerrors += 1

    print(("Done searching " + str(len(fileslist) - fileerrors) + " of " + str(len(fileslist))
           + " files for '" + search_term + "'"))
    if fileerrors > 0:
        print("--> Error opening " + str(fileerrors) + " file(s)")

    # results
    if len(foundinlist) > 0:
        print("\n\nSearch term found in " + str(len(foundinlist)) + " file(s):")
        for i in foundinlist:
            print(i)
        print("\n\nSearch term found on " + str(totallines) + " line(s):")
        for i in linelist:
            print(i)

        if results_file != "":
            results = ((["Searched " + str(len(fileslist) - fileerrors) + " of " + str(len(fileslist))
                         + " files for '" + search_term + "'"])
                       + ["\n\nSearch term found in " + str(len(foundinlist)) + " file(s):\n"]
                       + [i + "\n" for i in foundinlist]
                       + ["\n\nSearch term found on " + str(totallines) + " line(s)\n"]
                       + [i + "\n" for i in linelist]
                       )
            try:
                with open(results_file, "w") as re_file:
                    re_file.writelines(results)
                    print("\nResults file written to:\n" + results_file)
            except Exception as e:
                print("\nResults file could NOT be written! Check your results file path")
                print(e)
        return True

    else:
        print("\nNo matches found")
        if results_file != "":
            print("\nResults file NOT written (no matches)")
        return False


# run function
search_files(folder, search_term, case_sensitive, results_file)