The next generation of the Teknik Services. Written in ASP.NET. https://www.teknik.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

Program.cs 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. using nClam;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Data.Entity;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Mail;
  9. using System.Reflection;
  10. using System.Text;
  11. using Teknik.Areas.Transparency.Models;
  12. using Teknik.Areas.Upload.Models;
  13. using Teknik.Areas.Users.Models;
  14. using Teknik.Areas.Users.Utility;
  15. using Teknik.Configuration;
  16. using Teknik.Helpers;
  17. using Teknik.Models;
  18. namespace ServerMaint
  19. {
  20. public class Program
  21. {
  22. private static string currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
  23. private static string virusFile = Path.Combine(currentPath, "virusLogs.txt");
  24. private static string errorFile = Path.Combine(currentPath, "errorLogs.txt");
  25. private static string configPath = currentPath;
  26. private const string TAKEDOWN_REPORTER = "Teknik Automated System";
  27. public static event Action<string> OutputEvent;
  28. public static int Main(string[] args)
  29. {
  30. try
  31. {
  32. ArgumentOptions options = new ArgumentOptions();
  33. var parser = new CommandLine.Parser(config => config.HelpWriter = Console.Out);
  34. if (parser.ParseArguments(args, options))
  35. {
  36. if (!string.IsNullOrEmpty(options.Config))
  37. configPath = options.Config;
  38. if (Directory.Exists(configPath))
  39. {
  40. Config config = Config.Load(configPath);
  41. TeknikEntities db = new TeknikEntities();
  42. Output(string.Format("[{0}] Started Server Maintenance Process.", DateTime.Now));
  43. // Scan all the uploads for viruses, and remove the bad ones
  44. if (options.ScanUploads && config.UploadConfig.VirusScanEnable)
  45. {
  46. ScanUploads(config, db);
  47. }
  48. // Warns all the invalid accounts via email
  49. if (options.WarnAccounts)
  50. {
  51. WarnInvalidAccounts(config, db);
  52. }
  53. // Cleans all inactive users
  54. if (options.CleanUsers)
  55. {
  56. CleanAccounts(config, db, options.DaysBeforeDeletion);
  57. }
  58. // Cleans the email for unused accounts
  59. if (options.CleanEmails)
  60. {
  61. CleanEmail(config, db);
  62. }
  63. // Cleans all the git accounts that are unused
  64. if (options.CleanGit)
  65. {
  66. CleanGit(config, db);
  67. }
  68. // Generates a file for all of the user's last seen dates
  69. if (options.GenerateLastSeen)
  70. {
  71. GenerateLastSeen(config, db, options.LastSeenFile);
  72. }
  73. Output(string.Format("[{0}] Finished Server Maintenance Process.", DateTime.Now));
  74. return 0;
  75. }
  76. else
  77. {
  78. string msg = string.Format("[{0}] Config File does not exist.", DateTime.Now);
  79. File.AppendAllLines(errorFile, new List<string> { msg });
  80. Output(msg);
  81. }
  82. }
  83. else
  84. {
  85. Output(options.GetUsage());
  86. }
  87. }
  88. catch (Exception ex)
  89. {
  90. string msg = string.Format("[{0}] Exception: {1}", DateTime.Now, ex.GetFullMessage(true));
  91. File.AppendAllLines(errorFile, new List<string> { msg });
  92. Output(msg);
  93. }
  94. return -1;
  95. }
  96. public static void ScanUploads(Config config, TeknikEntities db)
  97. {
  98. Output(string.Format("[{0}] Started Virus Scan.", DateTime.Now));
  99. List<Upload> uploads = db.Uploads.ToList();
  100. // Initialize ClamAV
  101. ClamClient clam = new ClamClient(config.UploadConfig.ClamServer, config.UploadConfig.ClamPort);
  102. clam.MaxStreamSize = config.UploadConfig.MaxUploadSize;
  103. int totalCount = uploads.Count();
  104. int totalScans = 0;
  105. int totalClean = 0;
  106. int totalViruses = 0;
  107. foreach (Upload upload in uploads)
  108. {
  109. totalScans++;
  110. string subDir = upload.FileName[0].ToString();
  111. string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
  112. if (File.Exists(filePath))
  113. {
  114. // Read in the file
  115. byte[] data = File.ReadAllBytes(filePath);
  116. // If the IV is set, and Key is set, then decrypt it
  117. if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
  118. {
  119. // Decrypt the data
  120. data = AES.Decrypt(data, upload.Key, upload.IV);
  121. }
  122. // We have the data, let's scan it
  123. ClamScanResult scanResult = clam.SendAndScanFile(data);
  124. switch (scanResult.Result)
  125. {
  126. case ClamScanResults.Clean:
  127. totalClean++;
  128. string cleanMsg = string.Format("[{0}] Clean Scan: {1}/{2} Scanned | {3} - {4}", DateTime.Now, totalScans, totalCount, upload.Url, upload.FileName);
  129. Output(cleanMsg);
  130. break;
  131. case ClamScanResults.VirusDetected:
  132. totalViruses++;
  133. string msg = string.Format("[{0}] Virus Detected: {1} - {2} - {3}", DateTime.Now, upload.Url, upload.FileName, scanResult.InfectedFiles.First().VirusName);
  134. File.AppendAllLines(virusFile, new List<string> { msg });
  135. Output(msg);
  136. // Delete from the DB
  137. db.Uploads.Remove(upload);
  138. db.SaveChanges();
  139. // Delete the File
  140. if (File.Exists(filePath))
  141. {
  142. File.Delete(filePath);
  143. }
  144. break;
  145. case ClamScanResults.Error:
  146. string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, scanResult.RawResult);
  147. File.AppendAllLines(errorFile, new List<string> { errorMsg });
  148. Output(errorMsg);
  149. break;
  150. case ClamScanResults.Unknown:
  151. string unkMsg = string.Format("[{0}] Unknown Scan Result: {1}", DateTime.Now, scanResult.RawResult);
  152. File.AppendAllLines(errorFile, new List<string> { unkMsg });
  153. Output(unkMsg);
  154. break;
  155. }
  156. }
  157. }
  158. if (totalViruses > 0)
  159. {
  160. // Add to transparency report if any were found
  161. Takedown report = db.Takedowns.Create();
  162. report.Requester = TAKEDOWN_REPORTER;
  163. report.RequesterContact = config.SupportEmail;
  164. report.DateRequested = DateTime.Now;
  165. report.Reason = "Malware Found";
  166. report.ActionTaken = string.Format("{0} Uploads removed", totalViruses);
  167. report.DateActionTaken = DateTime.Now;
  168. db.Takedowns.Add(report);
  169. db.SaveChanges();
  170. }
  171. Output(string.Format("Scanning Complete. {0} Scanned | {1} Viruses Found | {2} Total Files", totalScans, totalViruses, totalCount));
  172. }
  173. public static void WarnInvalidAccounts(Config config, TeknikEntities db)
  174. {
  175. Output(string.Format("[{0}] Started Warning of Invalid Accounts.", DateTime.Now));
  176. List<string> invalidAccounts = GetInvalidAccounts(config, db);
  177. foreach (string account in invalidAccounts)
  178. {
  179. // Let's send them an email :D
  180. string email = UserHelper.GetUserEmailAddress(config, account);
  181. SmtpClient client = new SmtpClient();
  182. client.Host = config.ContactConfig.Host;
  183. client.Port = config.ContactConfig.Port;
  184. client.EnableSsl = config.ContactConfig.SSL;
  185. client.DeliveryMethod = SmtpDeliveryMethod.Network;
  186. client.UseDefaultCredentials = true;
  187. client.Credentials = new NetworkCredential(config.ContactConfig.Username, config.ContactConfig.Password);
  188. client.Timeout = 5000;
  189. MailMessage mail = new MailMessage(config.SupportEmail, email);
  190. mail.Subject = "Invalid Account Notice";
  191. mail.Body = string.Format(@"
  192. The account {0} does not meet the requirements for a valid username.
  193. The username must match the following Regex Pattern: {1}
  194. It must also be greater than or equal to {2} characters in length, and less than or equal to {3} characters in length.
  195. This email is to let you know that this account will be deleted in {4} days ({5}) in order to comply with the username restrictions. If you would like to keep your data, you should create a new account and transfer the data over to the new account.
  196. In order to make the process as easy as possible, you can reply to this email to ask for your current account to be renamed to another available account. This would keep all your data intact, and just require you to change all references to your email/git/user to the new username. If you wish to do this, please respond within {6} days ({7}).
  197. Thank you for your continued use of Teknik!
  198. - Teknik Administration", account, config.UserConfig.UsernameFilter, config.UserConfig.MinUsernameLength, config.UserConfig.MaxUsernameLength, 30, DateTime.Now.AddDays(30).ToShortDateString(), 15, DateTime.Now.AddDays(15).ToShortDateString());
  199. mail.BodyEncoding = UTF8Encoding.UTF8;
  200. mail.DeliveryNotificationOptions = DeliveryNotificationOptions.Never;
  201. client.Send(mail);
  202. }
  203. Output(string.Format("[{0}] Finished Warning of Invalid Accounts. {1} Accounts Warned.", DateTime.Now, invalidAccounts.Count));
  204. }
  205. public static void CleanAccounts(Config config, TeknikEntities db, int maxDays)
  206. {
  207. Output(string.Format("[{0}] Started Cleaning of Inactive/Invalid Users.", DateTime.Now));
  208. List<string> invalidAccounts = GetInvalidAccounts(config, db);
  209. List<string> inactiveAccounts = GetInactiveAccounts(config, db, maxDays);
  210. // Delete invalid accounts
  211. foreach (string account in invalidAccounts)
  212. {
  213. UserHelper.DeleteAccount(db, config, UserHelper.GetUser(db, account));
  214. }
  215. if (invalidAccounts.Count > 0)
  216. {
  217. // Add to transparency report if any users were removed
  218. Takedown report = db.Takedowns.Create();
  219. report.Requester = TAKEDOWN_REPORTER;
  220. report.RequesterContact = config.SupportEmail;
  221. report.DateRequested = DateTime.Now;
  222. report.Reason = "Username Invalid";
  223. report.ActionTaken = string.Format("{0} Accounts Removed", invalidAccounts.Count);
  224. report.DateActionTaken = DateTime.Now;
  225. db.Takedowns.Add(report);
  226. db.SaveChanges();
  227. }
  228. // Delete inactive accounts
  229. foreach (string account in inactiveAccounts)
  230. {
  231. UserHelper.DeleteAccount(db, config, UserHelper.GetUser(db, account));
  232. }
  233. if (invalidAccounts.Count > 0)
  234. {
  235. // Add to transparency report if any users were removed
  236. Takedown report = db.Takedowns.Create();
  237. report.Requester = TAKEDOWN_REPORTER;
  238. report.RequesterContact = config.SupportEmail;
  239. report.DateRequested = DateTime.Now;
  240. report.Reason = "Account Inactive";
  241. report.ActionTaken = string.Format("{0} Accounts Removed", inactiveAccounts.Count);
  242. report.DateActionTaken = DateTime.Now;
  243. db.Takedowns.Add(report);
  244. db.SaveChanges();
  245. }
  246. Output(string.Format("[{0}] Finished Cleaning of Inactive/Invalid Users. {1} Accounts Removed.", DateTime.Now, invalidAccounts.Count + inactiveAccounts.Count));
  247. }
  248. public static void CleanEmail(Config config, TeknikEntities db)
  249. {
  250. if (config.EmailConfig.Enabled)
  251. {
  252. Output(string.Format("[{0}] Started Cleaning of Orphaned Email Accounts.", DateTime.Now));
  253. List<User> curUsers = db.Users.ToList();
  254. int totalAccounts = 0;
  255. // Connect to hmailserver COM
  256. var app = new hMailServer.Application();
  257. app.Connect();
  258. app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password);
  259. var domain = app.Domains.ItemByName[config.EmailConfig.Domain];
  260. var accounts = domain.Accounts;
  261. for (int i = 0; i < accounts.Count; i++)
  262. {
  263. var account = accounts[i];
  264. bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username) == account.Address);
  265. bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r).ToLower() == account.Address.ToLower());
  266. if (!userExists && !isReserved)
  267. {
  268. // User doesn't exist, and it isn't reserved. Let's nuke it.
  269. UserHelper.DeleteUserEmail(config, account.Address);
  270. totalAccounts++;
  271. }
  272. }
  273. if (totalAccounts > 0)
  274. {
  275. // Add to transparency report if any users were removed
  276. Takedown report = db.Takedowns.Create();
  277. report.Requester = TAKEDOWN_REPORTER;
  278. report.RequesterContact = config.SupportEmail;
  279. report.DateRequested = DateTime.Now;
  280. report.Reason = "Orphaned Email Account";
  281. report.ActionTaken = string.Format("{0} Accounts Removed", totalAccounts);
  282. report.DateActionTaken = DateTime.Now;
  283. db.Takedowns.Add(report);
  284. db.SaveChanges();
  285. }
  286. Output(string.Format("[{0}] Finished Cleaning of Orphaned Email Accounts. {1} Accounts Removed.", DateTime.Now, totalAccounts));
  287. }
  288. }
  289. public static void CleanGit(Config config, TeknikEntities db)
  290. {
  291. if (config.GitConfig.Enabled)
  292. {
  293. Output(string.Format("[{0}] Started Cleaning of Orphaned Git Accounts.", DateTime.Now));
  294. List<User> curUsers = db.Users.ToList();
  295. int totalAccounts = 0;
  296. // We need to check the actual git database
  297. MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database);
  298. string sql = @"SELECT gogs.user.login_name AS login_name, gogs.user.lower_name AS username FROM gogs.user";
  299. var results = mySQL.Query(sql);
  300. if (results != null && results.Any())
  301. {
  302. foreach (var account in results)
  303. {
  304. bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username).ToLower() == account["login_name"].ToString().ToLower());
  305. bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r) == account["login_name"].ToString().ToLower());
  306. if (!userExists && !isReserved)
  307. {
  308. UserHelper.DeleteUserGit(config, account["username"].ToString());
  309. totalAccounts++;
  310. }
  311. }
  312. }
  313. if (totalAccounts > 0)
  314. {
  315. // Add to transparency report if any users were removed
  316. Takedown report = db.Takedowns.Create();
  317. report.Requester = TAKEDOWN_REPORTER;
  318. report.RequesterContact = config.SupportEmail;
  319. report.DateRequested = DateTime.Now;
  320. report.Reason = "Orphaned Git Account";
  321. report.ActionTaken = string.Format("{0} Accounts Removed", totalAccounts);
  322. report.DateActionTaken = DateTime.Now;
  323. db.Takedowns.Add(report);
  324. db.SaveChanges();
  325. }
  326. Output(string.Format("[{0}] Finished Cleaning of Orphaned Git Accounts. {1} Accounts Removed.", DateTime.Now, totalAccounts));
  327. }
  328. }
  329. public static void GenerateLastSeen(Config config, TeknikEntities db, string fileName)
  330. {
  331. Output(string.Format("[{0}] Started Generation of Last Activity List.", DateTime.Now));
  332. List<User> curUsers = db.Users.ToList();
  333. StringBuilder sb = new StringBuilder();
  334. sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity");
  335. foreach (User user in curUsers)
  336. {
  337. sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}",
  338. user.Username,
  339. UserHelper.GetLastAccountActivity(db, config, user).ToString("g"),
  340. user.JoinDate.ToString("g"),
  341. user.LastSeen.ToString("g"),
  342. UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g"),
  343. UserHelper.UserGitLastActive(config, user.Username).ToString("g")));
  344. }
  345. string dir = Path.GetDirectoryName(fileName);
  346. if (!Directory.Exists(dir))
  347. Directory.CreateDirectory(dir);
  348. File.WriteAllText(fileName, sb.ToString());
  349. Output(string.Format("[{0}] Finished Generating Last Activity List.", DateTime.Now));
  350. }
  351. public static List<string> GetInvalidAccounts(Config config, TeknikEntities db)
  352. {
  353. List<string> foundUsers = new List<string>();
  354. List<User> curUsers = db.Users.ToList();
  355. foreach (User user in curUsers)
  356. {
  357. // If the username is reserved, don't worry about it
  358. if (UserHelper.UsernameReserved(config, user.Username))
  359. {
  360. continue;
  361. }
  362. // If the username is invalid, let's add it to the list
  363. if (!UserHelper.ValidUsername(config, user.Username))
  364. {
  365. foundUsers.Add(user.Username);
  366. continue;
  367. }
  368. }
  369. return foundUsers;
  370. }
  371. public static List<string> GetInactiveAccounts(Config config, TeknikEntities db, int maxDays)
  372. {
  373. List<string> foundUsers = new List<string>();
  374. List<User> curUsers = db.Users.ToList();
  375. foreach (User user in curUsers)
  376. {
  377. // If the username is reserved, don't worry about it
  378. if (UserHelper.UsernameReserved(config, user.Username))
  379. {
  380. continue;
  381. }
  382. #region Inactivity Finding
  383. DateTime lastActivity = UserHelper.GetLastAccountActivity(db, config, user);
  384. TimeSpan inactiveTime = DateTime.Now.Subtract(lastActivity);
  385. // If older than max days, check their current usage
  386. if (inactiveTime >= new TimeSpan(maxDays, 0, 0, 0, 0))
  387. {
  388. // Check the user's usage of the service.
  389. bool noData = true;
  390. // Any blog comments?
  391. var blogCom = db.BlogComments.Include("Users").Where(c => c.UserId == user.UserId);
  392. noData &= !(blogCom != null && blogCom.Any());
  393. // Any blog posts?
  394. var blogPosts = db.BlogPosts.Include("Blog").Include("Blog.Users").Where(p => p.Blog.UserId == user.UserId);
  395. noData &= !(blogPosts != null && blogPosts.Any());
  396. // Any podcast comments?
  397. var podCom = db.PodcastComments.Include("Users").Where(p => p.UserId == user.UserId);
  398. noData &= !(podCom != null && podCom.Any());
  399. // Any email?
  400. if (config.EmailConfig.Enabled)
  401. {
  402. var app = new hMailServer.Application();
  403. app.Connect();
  404. app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password);
  405. try
  406. {
  407. var domain = app.Domains.ItemByName[config.EmailConfig.Domain];
  408. var account = domain.Accounts.ItemByAddress[UserHelper.GetUserEmailAddress(config, user.Username)];
  409. noData &= ((account.Messages.Count == 0) && ((int)account.Size == 0));
  410. }
  411. catch { }
  412. }
  413. // Any git repos?
  414. if (config.GitConfig.Enabled)
  415. {
  416. string email = UserHelper.GetUserEmailAddress(config, user.Username);
  417. // We need to check the actual git database
  418. MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database);
  419. string sql = @"SELECT * FROM gogs.repository
  420. LEFT JOIN gogs.action ON gogs.user.id = gogs.action.act_user_id
  421. WHERE gogs.user.login_name = {0}";
  422. var results = mySQL.Query(sql, new object[] { email });
  423. noData &= !(results != null && results.Any());
  424. }
  425. if (noData)
  426. {
  427. // They have no data, so safe to delete them.
  428. foundUsers.Add(user.Username);
  429. }
  430. continue;
  431. }
  432. #endregion
  433. }
  434. return foundUsers;
  435. }
  436. public static void Output(string message)
  437. {
  438. Console.WriteLine(message);
  439. if (OutputEvent != null)
  440. {
  441. OutputEvent(message);
  442. }
  443. }
  444. }
  445. }