The next generation of the Teknik Services. Written in ASP.NET. Fork for blog tags.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

Program.cs 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. using nClam;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Data.Entity;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Mail;
  9. using System.Reflection;
  10. using System.Text;
  11. using Teknik.Areas.Transparency.Models;
  12. using Teknik.Areas.Upload.Models;
  13. using Teknik.Areas.Users.Models;
  14. using Teknik.Areas.Users.Utility;
  15. using Teknik.Configuration;
  16. using Teknik.Helpers;
  17. using Teknik.Models;
  18. namespace ServerMaint
  19. {
  20. public class Program
  21. {
  22. private static string currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
  23. private static string virusFile = Path.Combine(currentPath, "virusLogs.txt");
  24. private static string errorFile = Path.Combine(currentPath, "errorLogs.txt");
  25. private static string configPath = currentPath;
  26. private const string TAKEDOWN_REPORTER = "Teknik Automated System";
  27. public static event Action<string> OutputEvent;
  28. public static int Main(string[] args)
  29. {
  30. try
  31. {
  32. ArgumentOptions options = new ArgumentOptions();
  33. var parser = new CommandLine.Parser(config => config.HelpWriter = Console.Out);
  34. if (parser.ParseArguments(args, options))
  35. {
  36. if (!string.IsNullOrEmpty(options.Config))
  37. configPath = options.Config;
  38. if (Directory.Exists(configPath))
  39. {
  40. Config config = Config.Load(configPath);
  41. TeknikEntities db = new TeknikEntities();
  42. Output(string.Format("[{0}] Started Server Maintenance Process.", DateTime.Now));
  43. // Scan all the uploads for viruses, and remove the bad ones
  44. if (options.ScanUploads && config.UploadConfig.VirusScanEnable)
  45. {
  46. ScanUploads(config, db);
  47. }
  48. // Cleans all inactive users
  49. if (options.CleanUsers)
  50. {
  51. CleanUsers(config, db, options.DaysBeforeDeletion);
  52. }
  53. // Cleans the email for unused accounts
  54. if (options.CleanEmails)
  55. {
  56. CleanEmail(config, db);
  57. }
  58. // Cleans all the git accounts that are unused
  59. if (options.CleanGit)
  60. {
  61. CleanGit(config, db);
  62. }
  63. // Generates a file for all of the user's last seen dates
  64. if (options.GenerateLastSeen)
  65. {
  66. GenerateLastSeen(config, db, options.LastSeenFile);
  67. }
  68. Output(string.Format("[{0}] Finished Server Maintenance Process.", DateTime.Now));
  69. return 0;
  70. }
  71. else
  72. {
  73. string msg = string.Format("[{0}] Config File does not exist.", DateTime.Now);
  74. File.AppendAllLines(errorFile, new List<string> { msg });
  75. Output(msg);
  76. }
  77. }
  78. else
  79. {
  80. Output(options.GetUsage());
  81. }
  82. }
  83. catch (Exception ex)
  84. {
  85. string msg = string.Format("[{0}] Exception: {1}", DateTime.Now, ex.GetFullMessage(true));
  86. File.AppendAllLines(errorFile, new List<string> { msg });
  87. Output(msg);
  88. }
  89. return -1;
  90. }
  91. public static void ScanUploads(Config config, TeknikEntities db)
  92. {
  93. Output(string.Format("[{0}] Started Virus Scan.", DateTime.Now));
  94. List<Upload> uploads = db.Uploads.ToList();
  95. // Initialize ClamAV
  96. ClamClient clam = new ClamClient(config.UploadConfig.ClamServer, config.UploadConfig.ClamPort);
  97. clam.MaxStreamSize = config.UploadConfig.MaxUploadSize;
  98. int totalCount = uploads.Count();
  99. int totalScans = 0;
  100. int totalClean = 0;
  101. int totalViruses = 0;
  102. foreach (Upload upload in uploads)
  103. {
  104. totalScans++;
  105. string subDir = upload.FileName[0].ToString();
  106. string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
  107. if (File.Exists(filePath))
  108. {
  109. // Read in the file
  110. byte[] data = File.ReadAllBytes(filePath);
  111. // If the IV is set, and Key is set, then decrypt it
  112. if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
  113. {
  114. // Decrypt the data
  115. data = AES.Decrypt(data, upload.Key, upload.IV);
  116. }
  117. // We have the data, let's scan it
  118. ClamScanResult scanResult = clam.SendAndScanFile(data);
  119. switch (scanResult.Result)
  120. {
  121. case ClamScanResults.Clean:
  122. totalClean++;
  123. string cleanMsg = string.Format("[{0}] Clean Scan: {1}/{2} Scanned | {3} - {4}", DateTime.Now, totalScans, totalCount, upload.Url, upload.FileName);
  124. Output(cleanMsg);
  125. break;
  126. case ClamScanResults.VirusDetected:
  127. totalViruses++;
  128. string msg = string.Format("[{0}] Virus Detected: {1} - {2} - {3}", DateTime.Now, upload.Url, upload.FileName, scanResult.InfectedFiles.First().VirusName);
  129. File.AppendAllLines(virusFile, new List<string> { msg });
  130. Output(msg);
  131. // Delete from the DB
  132. db.Uploads.Remove(upload);
  133. db.SaveChanges();
  134. // Delete the File
  135. if (File.Exists(filePath))
  136. {
  137. File.Delete(filePath);
  138. }
  139. break;
  140. case ClamScanResults.Error:
  141. string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, scanResult.RawResult);
  142. File.AppendAllLines(errorFile, new List<string> { errorMsg });
  143. Output(errorMsg);
  144. break;
  145. case ClamScanResults.Unknown:
  146. string unkMsg = string.Format("[{0}] Unknown Scan Result: {1}", DateTime.Now, scanResult.RawResult);
  147. File.AppendAllLines(errorFile, new List<string> { unkMsg });
  148. Output(unkMsg);
  149. break;
  150. }
  151. }
  152. }
  153. if (totalViruses > 0)
  154. {
  155. // Add to transparency report if any were found
  156. Takedown report = db.Takedowns.Create();
  157. report.Requester = TAKEDOWN_REPORTER;
  158. report.RequesterContact = config.SupportEmail;
  159. report.DateRequested = DateTime.Now;
  160. report.Reason = "Malware Found";
  161. report.ActionTaken = string.Format("{0} Uploads removed", totalViruses);
  162. report.DateActionTaken = DateTime.Now;
  163. db.Takedowns.Add(report);
  164. db.SaveChanges();
  165. }
  166. Output(string.Format("Scanning Complete. {0} Scanned | {1} Viruses Found | {2} Total Files", totalScans, totalViruses, totalCount));
  167. }
  168. public static void CleanUsers(Config config, TeknikEntities db, int maxDays)
  169. {
  170. int totalUsers = 0;
  171. Output(string.Format("[{0}] Started Cleaning of Inactive Users.", DateTime.Now));
  172. List<User> curUsers = db.Users.ToList();
  173. foreach (User user in curUsers)
  174. {
  175. // If the username is reserved, don't clean it
  176. if (UserHelper.UsernameReserved(config, user.Username))
  177. {
  178. continue;
  179. }
  180. // If the username is invalid, let's clean the sucker, data and all
  181. if (!UserHelper.ValidUsername(config, user.Username))
  182. {
  183. UserHelper.DeleteAccount(db, config, user);
  184. continue;
  185. }
  186. #region Inactivity Cleaning
  187. DateTime lastActivity = UserHelper.GetLastAccountActivity(db, config, user);
  188. TimeSpan inactiveTime = DateTime.Now.Subtract(lastActivity);
  189. // If older than max days, check their current usage
  190. if (inactiveTime >= new TimeSpan(maxDays, 0, 0, 0, 0))
  191. {
  192. // Check the user's usage of the service.
  193. bool noData = true;
  194. // Any blog comments?
  195. var blogCom = db.BlogComments.Include("Users").Where(c => c.UserId == user.UserId);
  196. noData &= !(blogCom != null && blogCom.Any());
  197. // Any blog posts?
  198. var blogPosts = db.BlogPosts.Include("Blog").Include("Blog.Users").Where(p => p.Blog.UserId == user.UserId);
  199. noData &= !(blogPosts != null && blogPosts.Any());
  200. // Any podcast comments?
  201. var podCom = db.PodcastComments.Include("Users").Where(p => p.UserId == user.UserId);
  202. noData &= !(podCom != null && podCom.Any());
  203. // Any email?
  204. if (config.EmailConfig.Enabled)
  205. {
  206. var app = new hMailServer.Application();
  207. app.Connect();
  208. app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password);
  209. try
  210. {
  211. var domain = app.Domains.ItemByName[config.EmailConfig.Domain];
  212. var account = domain.Accounts.ItemByAddress[UserHelper.GetUserEmailAddress(config, user.Username)];
  213. noData &= ((account.Messages.Count == 0) && ((int)account.Size == 0));
  214. }
  215. catch { }
  216. }
  217. // Any git repos?
  218. if (config.GitConfig.Enabled)
  219. {
  220. string email = UserHelper.GetUserEmailAddress(config, user.Username);
  221. // We need to check the actual git database
  222. MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database);
  223. string sql = @"SELECT * FROM gogs.repository
  224. LEFT JOIN gogs.action ON gogs.user.id = gogs.action.act_user_id
  225. WHERE gogs.user.login_name = {0}";
  226. var results = mySQL.Query(sql, new object[] { email });
  227. noData &= !(results != null && results.Any());
  228. }
  229. if (noData)
  230. {
  231. // They have no data, so safe to delete them.
  232. UserHelper.DeleteAccount(db, config, UserHelper.GetUser(db, user.Username));
  233. totalUsers++;
  234. }
  235. continue;
  236. }
  237. #endregion
  238. }
  239. if (totalUsers > 0)
  240. {
  241. // Add to transparency report if any users were removed
  242. Takedown report = db.Takedowns.Create();
  243. report.Requester = TAKEDOWN_REPORTER;
  244. report.RequesterContact = config.SupportEmail;
  245. report.DateRequested = DateTime.Now;
  246. report.Reason = "User Inactive";
  247. report.ActionTaken = string.Format("{0} Users Removed", totalUsers);
  248. report.DateActionTaken = DateTime.Now;
  249. db.Takedowns.Add(report);
  250. db.SaveChanges();
  251. }
  252. Output(string.Format("[{0}] Finished Cleaning of Inactive Users. {1} Users Removed.", DateTime.Now, totalUsers));
  253. }
  254. public static void CleanEmail(Config config, TeknikEntities db)
  255. {
  256. if (config.EmailConfig.Enabled)
  257. {
  258. Output(string.Format("[{0}] Started Cleaning of Orphaned Email Accounts.", DateTime.Now));
  259. List<User> curUsers = db.Users.ToList();
  260. int totalAccounts = 0;
  261. // Connect to hmailserver COM
  262. var app = new hMailServer.Application();
  263. app.Connect();
  264. app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password);
  265. var domain = app.Domains.ItemByName[config.EmailConfig.Domain];
  266. var accounts = domain.Accounts;
  267. for (int i = 0; i < accounts.Count; i++)
  268. {
  269. var account = accounts[i];
  270. bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username) == account.Address);
  271. bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r).ToLower() == account.Address.ToLower());
  272. if (!userExists && !isReserved)
  273. {
  274. // User doesn't exist, and it isn't reserved. Let's nuke it.
  275. UserHelper.DeleteUserEmail(config, account.Address);
  276. totalAccounts++;
  277. }
  278. }
  279. if (totalAccounts > 0)
  280. {
  281. // Add to transparency report if any users were removed
  282. Takedown report = db.Takedowns.Create();
  283. report.Requester = TAKEDOWN_REPORTER;
  284. report.RequesterContact = config.SupportEmail;
  285. report.DateRequested = DateTime.Now;
  286. report.Reason = "Orphaned Email Account";
  287. report.ActionTaken = string.Format("{0} Accounts Removed", totalAccounts);
  288. report.DateActionTaken = DateTime.Now;
  289. db.Takedowns.Add(report);
  290. db.SaveChanges();
  291. }
  292. Output(string.Format("[{0}] Finished Cleaning of Orphaned Email Accounts. {1} Accounts Removed.", DateTime.Now, totalAccounts));
  293. }
  294. }
  295. public static void CleanGit(Config config, TeknikEntities db)
  296. {
  297. if (config.GitConfig.Enabled)
  298. {
  299. Output(string.Format("[{0}] Started Cleaning of Orphaned Git Accounts.", DateTime.Now));
  300. List<User> curUsers = db.Users.ToList();
  301. int totalAccounts = 0;
  302. // We need to check the actual git database
  303. MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database);
  304. string sql = @"SELECT gogs.user.login_name AS login_name, gogs.user.lower_name AS username FROM gogs.user";
  305. var results = mySQL.Query(sql);
  306. if (results != null && results.Any())
  307. {
  308. foreach (var account in results)
  309. {
  310. bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username).ToLower() == account["login_name"].ToString().ToLower());
  311. bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r) == account["login_name"].ToString().ToLower());
  312. if (!userExists && !isReserved)
  313. {
  314. UserHelper.DeleteUserGit(config, account["username"].ToString());
  315. totalAccounts++;
  316. }
  317. }
  318. }
  319. if (totalAccounts > 0)
  320. {
  321. // Add to transparency report if any users were removed
  322. Takedown report = db.Takedowns.Create();
  323. report.Requester = TAKEDOWN_REPORTER;
  324. report.RequesterContact = config.SupportEmail;
  325. report.DateRequested = DateTime.Now;
  326. report.Reason = "Orphaned Git Account";
  327. report.ActionTaken = string.Format("{0} Accounts Removed", totalAccounts);
  328. report.DateActionTaken = DateTime.Now;
  329. db.Takedowns.Add(report);
  330. db.SaveChanges();
  331. }
  332. Output(string.Format("[{0}] Finished Cleaning of Orphaned Git Accounts. {1} Accounts Removed.", DateTime.Now, totalAccounts));
  333. }
  334. }
  335. public static void GenerateLastSeen(Config config, TeknikEntities db, string fileName)
  336. {
  337. Output(string.Format("[{0}] Started Generation of Last Activity List.", DateTime.Now));
  338. List<User> curUsers = db.Users.ToList();
  339. StringBuilder sb = new StringBuilder();
  340. sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity");
  341. foreach (User user in curUsers)
  342. {
  343. sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}",
  344. user.Username,
  345. UserHelper.GetLastAccountActivity(db, config, user).ToString("g"),
  346. user.JoinDate.ToString("g"),
  347. user.LastSeen.ToString("g"),
  348. UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g"),
  349. UserHelper.UserGitLastActive(config, user.Username).ToString("g")));
  350. }
  351. string dir = Path.GetDirectoryName(fileName);
  352. if (!Directory.Exists(dir))
  353. Directory.CreateDirectory(dir);
  354. File.WriteAllText(fileName, sb.ToString());
  355. Output(string.Format("[{0}] Finished Generating Last Activity List.", DateTime.Now));
  356. }
  357. public static void Output(string message)
  358. {
  359. Console.WriteLine(message);
  360. if (OutputEvent != null)
  361. {
  362. OutputEvent(message);
  363. }
  364. }
  365. }
  366. }