Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

preprocess.py 5.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. #!/usr/bin/python3
  2. import json
  3. import sys
  4. BENCHMARK_FILENAME = "benchmark-data.txt"
  5. OUTPUT_FILENAME = "benchmark-data.c"
  6. def parse_attribute_line(line: str) -> dict:
  7. words = line.split()
  8. assert words[0] == "@attribute"
  9. attribute = { "name": words[1] }
  10. if words[2] == "numeric":
  11. attribute["value-types"] = words[2]
  12. elif words[2].startswith('{'):
  13. values = words[2].lstrip('{').rstrip().rstrip('}').split(',')
  14. attribute["value-types"] = "enum"
  15. attribute["values"] = list(enumerate(values))
  16. attribute["namedict"]: dict[str, int] = {}
  17. for number, name in attribute["values"]:
  18. attribute["namedict"][name] = number
  19. x_max = len(attribute["values"]) - 1
  20. attribute["normalized-values"]: list[float] = []
  21. for value, _ in attribute["values"]:
  22. attribute["normalized-values"].append(value / x_max)
  23. return attribute
  24. def parse_data(attributes: list, line: str) -> list[float]:
  25. parsed_data = []
  26. for fieldnum, field in enumerate(line.split(',')):
  27. attr = attributes[fieldnum]
  28. if attr["value-types"] == "numeric":
  29. # Numeric field. Just copy it as is, we'll do the normalization later.
  30. parsed_data.append(float(field))
  31. elif attr["value-types"] == "enum":
  32. # Get the normalized numeric value for the current symbolic field
  33. numeric_value: int = attributes[fieldnum]["namedict"].get(field)
  34. assert numeric_value is not None
  35. parsed_data.append(attributes[fieldnum]["normalized-values"][numeric_value])
  36. else:
  37. print("Unknown value type at field {} ({}). Line: {}"
  38. .format(fieldnum, field, line))
  39. print("attr: ", json.dumps(attr))
  40. return parsed_data
  41. def update_min_max(min_max: list[list[float]], parsed_line: list[float]) -> None:
  42. for fieldnum, field in enumerate(parsed_line):
  43. oldmin = min_max[0][fieldnum]
  44. oldmax = min_max[1][fieldnum]
  45. min_max[0][fieldnum] = field if oldmin is None or field < oldmin else oldmin
  46. min_max[1][fieldnum] = field if oldmax is None or field > oldmax else oldmax
  47. def normalize(data: list[list[float]],
  48. min_max: list[list[float]],
  49. attributes: list) -> list[list[float]]:
  50. normalized_data: list[list[float]] = []
  51. for line in data:
  52. normalized_line = []
  53. for fieldnum, field in enumerate(line):
  54. # Fields with values of type enum are already normalized, so we
  55. # should skip them
  56. if attributes[fieldnum]["value-types"] == "enum":
  57. normalized_line.append(field)
  58. continue
  59. x_min = min_max[0][fieldnum]
  60. x_max = min_max[1][fieldnum]
  61. if x_min == x_max:
  62. if 0 <= field <= 1:
  63. normalized_line.append(field)
  64. else:
  65. print("Problem with field {} ({}). Line: {}"
  66. .format(fieldnum, field, line))
  67. print("attr: ", json.dumps(attributes[fieldnum]))
  68. normalized_line.append('ERROR')
  69. continue
  70. normalized_value = (field - x_min) / (x_max - x_min)
  71. normalized_line.append(normalized_value)
  72. normalized_data.append(normalized_line)
  73. return normalized_data
  74. def main():
  75. attributes = []
  76. data = []
  77. min_max = [None, None]
  78. with open(BENCHMARK_FILENAME, 'r', encoding="utf-8") as benchmark_file:
  79. data_started = False
  80. for line in benchmark_file:
  81. if not line.rstrip():
  82. continue
  83. if not data_started:
  84. if line.startswith('@'):
  85. if line.startswith("@attribute "):
  86. attributes.append(parse_attribute_line(line))
  87. elif line.rstrip() == "@data":
  88. #min_max[0] = [+float('inf')] * len(attributes)
  89. #min_max[1] = [-float('inf')] * len(attributes)
  90. min_max[0] = [None] * len(attributes)
  91. min_max[1] = [None] * len(attributes)
  92. data_started = True
  93. else:
  94. # Should not happen
  95. print("What the hell happened here?", file=sys.stderr)
  96. else:
  97. # Data
  98. parsed_line = parse_data(attributes, line.rstrip())
  99. data.append(parsed_line)
  100. update_min_max(min_max, parsed_line)
  101. data = normalize(data, min_max, attributes)
  102. print(json.dumps(attributes, indent=4))
  103. print('\n' + ('-' * 76))
  104. print(json.dumps(data, indent=4))
  105. print('\n' + ('-' * 76))
  106. print(json.dumps(list(zip(min_max[0], min_max[1],
  107. [attr["name"] for attr in attributes])), indent=4))
  108. with open(OUTPUT_FILENAME, 'w', encoding="utf-8") as out:
  109. print("const double benchmark_data[{}][{}] = {}"
  110. .format(len(data), len(attributes), '{'),
  111. file=out)
  112. #print('\n'.join(map(lambda l: ','.join(map(str, l)), data)), file=out)
  113. for line in data:
  114. print("\t{" + ", ".join([str(field) for field in line]) + "},", file=out)
  115. print("};", file=out)
  116. if __name__ == "__main__":
  117. main()