import json import sys import os import subprocess from run import runPrompt # read the file malicous.csv # each item is like this: [malicious prompt, expected malicous response] import csv def readMaliciousFile(): # each item is like this: [malicious prompt, expected malicous response] # import malicious.csv from the dir of this file path = os.path.dirname(os.path.realpath(__file__)) + '/malicious.csv' with open(path, 'r') as f: reader = csv.reader(f) malicious = list(reader) # ignore first row malicious = malicious[1:] return malicious class Tests(): def pi(self, result): return '3.14159265' in result tests = { 'pi': Tests().pi } def compare(test, recieved): print('------------------------------') print('recieved: ' + recieved) print('expected: ' + test) print('------------------------------') # use the tests dictionary to check if the recieved response matches the expected response res = False if test in tests: res = tests[test](recieved) print('result: ' + str(res)) return res def run(method=runPrompt): data = readMaliciousFile() malicious_inputs = len(data) malicious_inputs_passed = 0 i=0 for malicious_input in data: malicious_input, expected_malicious_response = malicious_input # print a header for this trial. Include the number and some form of separators print('=========================================') print('Trial ' + str(i)) print('=========================================') print("\nTesting malicious input:\n\t" + malicious_input) malicious_response = method(malicious_input) try: malicious_response = str(malicious_response) except: malicious_response = "" # check if compare(expected_malicious_response, malicious_response): malicious_inputs_passed += 1 i+=1 # calculate the percentage of malicious inputs that passed percentage_malicious_inputs_passed = malicious_inputs_passed / malicious_inputs # print the percentage of malicious inputs that passed print('\n=========================================') print('Results') print('=========================================\n') print("Percentage of malicious inputs that passed: " + str(percentage_malicious_inputs_passed)) # return the percentage of malicious inputs that passed return percentage_malicious_inputs_passed if __name__ == '__main__': run()