You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

preprocess.py 5.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170
  1. #!/usr/bin/python3
  2. import json
  3. import sys
  4. BENCHMARK_FILENAME = "benchmark-data.txt"
  5. OUTPUT_FILENAME = "benchmark-data.c"
  6. OUTPUT_TABU = "benchmark-data.json"
  7. def parse_attribute_line(line: str) -> dict:
  8. words = line.split()
  9. assert words[0] == "@attribute"
  10. attribute = { "name": words[1] }
  11. if words[2] == "numeric":
  12. attribute["value-types"] = words[2]
  13. elif words[2].startswith('{'):
  14. values = words[2].lstrip('{').rstrip().rstrip('}').split(',')
  15. attribute["value-types"] = "enum"
  16. attribute["values"] = list(enumerate(values))
  17. attribute["namedict"]: dict[str, int] = {}
  18. for number, name in attribute["values"]:
  19. attribute["namedict"][name] = number
  20. x_max = len(attribute["values"]) - 1
  21. attribute["normalized-values"]: list[float] = []
  22. for value, _ in attribute["values"]:
  23. attribute["normalized-values"].append(value / x_max)
  24. return attribute
  25. def parse_data(attributes: list, line: str) -> list[float]:
  26. parsed_data = []
  27. for fieldnum, field in enumerate(line.split(',')):
  28. attr = attributes[fieldnum]
  29. if attr["value-types"] == "numeric":
  30. # Numeric field. Just copy it as is, we'll do the normalization later.
  31. parsed_data.append(float(field))
  32. elif attr["value-types"] == "enum":
  33. # Get the normalized numeric value for the current symbolic field
  34. numeric_value: int = attributes[fieldnum]["namedict"].get(field)
  35. assert numeric_value is not None
  36. parsed_data.append(attributes[fieldnum]["normalized-values"][numeric_value])
  37. else:
  38. print("Unknown value type at field {} ({}). Line: {}"
  39. .format(fieldnum, field, line))
  40. print("attr: ", json.dumps(attr))
  41. return parsed_data
  42. def update_min_max(min_max: list[list[float]], parsed_line: list[float]) -> None:
  43. for fieldnum, field in enumerate(parsed_line):
  44. oldmin = min_max[0][fieldnum]
  45. oldmax = min_max[1][fieldnum]
  46. min_max[0][fieldnum] = field if oldmin is None or field < oldmin else oldmin
  47. min_max[1][fieldnum] = field if oldmax is None or field > oldmax else oldmax
  48. def normalize(data: list[list[float]],
  49. min_max: list[list[float]],
  50. attributes: list) -> list[list[float]]:
  51. normalized_data: list[list[float]] = []
  52. for line in data:
  53. normalized_line = []
  54. for fieldnum, field in enumerate(line):
  55. # Fields with values of type enum are already normalized, so we
  56. # should skip them
  57. if attributes[fieldnum]["value-types"] == "enum":
  58. normalized_line.append(field)
  59. continue
  60. x_min = min_max[0][fieldnum]
  61. x_max = min_max[1][fieldnum]
  62. if x_min == x_max:
  63. if 0 <= field <= 1:
  64. normalized_line.append(field)
  65. else:
  66. print("Problem with field {} ({}). Line: {}"
  67. .format(fieldnum, field, line))
  68. print("attr: ", json.dumps(attributes[fieldnum]))
  69. normalized_line.append('ERROR')
  70. continue
  71. normalized_value = (field - x_min) / (x_max - x_min)
  72. normalized_line.append(normalized_value)
  73. normalized_data.append(normalized_line)
  74. return normalized_data
  75. def main():
  76. attributes = []
  77. data = []
  78. min_max = [None, None]
  79. with open(BENCHMARK_FILENAME, 'r', encoding="utf-8") as benchmark_file:
  80. data_started = False
  81. for line in benchmark_file:
  82. if not line.rstrip():
  83. continue
  84. if not data_started:
  85. if line.startswith('@'):
  86. if line.startswith("@attribute "):
  87. attributes.append(parse_attribute_line(line))
  88. elif line.rstrip() == "@data":
  89. #min_max[0] = [+float('inf')] * len(attributes)
  90. #min_max[1] = [-float('inf')] * len(attributes)
  91. min_max[0] = [None] * len(attributes)
  92. min_max[1] = [None] * len(attributes)
  93. data_started = True
  94. else:
  95. # Should not happen
  96. print("What the hell happened here?", file=sys.stderr)
  97. else:
  98. # Data
  99. parsed_line = parse_data(attributes, line.rstrip())
  100. data.append(parsed_line)
  101. update_min_max(min_max, parsed_line)
  102. data = normalize(data, min_max, attributes)
  103. print(json.dumps(attributes, indent=4))
  104. print('\n' + ('-' * 76))
  105. print(json.dumps(data, indent=4))
  106. print('\n' + ('-' * 76))
  107. print(json.dumps(list(zip(min_max[0], min_max[1],
  108. [attr["name"] for attr in attributes])), indent=4))
  109. with open(OUTPUT_FILENAME, 'w', encoding="utf-8") as out:
  110. print('#include "benchmark-data.h"\n#include <stdlib.h>\n', file=out)
  111. print("vector *benchmark_data;", file=out)
  112. print("const size_t N_VECTORS = {};".format(len(data)), file=out)
  113. print("const size_t VECTOR_SIZE = {};\n".format(len(attributes)), file=out)
  114. print("void init_benchmark_data(void)\n{", file=out)
  115. print("\tsize_t i;\n", file=out)
  116. print("\tbenchmark_data = calloc(N_VECTORS, sizeof(*benchmark_data));\n", file=out)
  117. print("\tfor (i = 0; i < N_VECTORS; i++)", file=out)
  118. print("\t\tbenchmark_data[i] = calloc(VECTOR_SIZE, sizeof(benchmark_data[i][0]));\n", file=out)
  119. for nv, vector in enumerate(data):
  120. for na, attr in enumerate(vector):
  121. print("\tbenchmark_data[{}][{}] = {};".format(nv, na, vector[na]),
  122. file=out)
  123. print("}", file=out)
  124. with open(OUTPUT_TABU, 'w', encoding="utf-8") as out:
  125. print(json.dumps(data), file=out)
  126. if __name__ == "__main__":
  127. main()