Diff -- /var/www/vhosts/3dshawn.com/site1/_file

1

#!/usr/bin/perl

2

3

######################################################################################################################

4

######################################################################################################################

5

# _____ __ __ ___ _ __

6

# / ___/ __ __ _____ / /_ ___ ____ ___ / |/ /____ ____ (_)/ /_ ____ _____

7

# \__ \ / / / // ___// __// _ \ / __ `__ \ / /|_/ // __ \ / __ \ / // __// __ \ / ___/

8

# ___/ // /_/ /(__ )/ /_ / __// / / / / / / / / // /_/ // / / // // /_ / /_/ // /

9

# /____/ \__, //____/ \__/ \___//_/ /_/ /_/ /_/ /_/ \____//_/ /_//_/ \__/ \____//_/

10

# /____/

11

#

12

# Program:(V2.1) Developed By:(Shawn Pickering) Email:(shawn@shawnpickering.com)

13

######################################################################################################################

14

######################################################################################################################

15

16

#Make sure you start this program with a user that has high enough permissions to copy the files over

17

#To run this in the background run this command: nohup ./_file_monitor.cgi &

18

#To kill the process get the process id: ps -ax|grep _file_monitor.cgi

19

#Then run this command: kill -9 process_id_goes_here

20

21

22

23

############################################################

24

## Modules

25

############################################################

26

use strict;

27

use DBI;

28

29

30

31

#############################################################

32

## Variables

33

#############################################################

34

# Schema Change Software MySQL database configurations

35

my $db_host = "98.7.189.33";

36

my $db_name = "SYSTEM_MONITOR";

37

my $db_port = "3306";

38

my $db_user = "root";

39

my $db_pass = 'mariapass1';

40

41

#Set the number of seconds to wait between scanning the directory for changes

42

my $pause_seconds = '60';

43

44

#Path and filename to save timestamps in (no need to change this)

45

my $monitor_file = '.files.dat';

46

47

#Save all errors to this error file

48

my $error_file = '.errors.dat';

49

50

#Folder a copy of all changed files that are scanned, this is the version control area

51

my $version_archive = "FILES/";

52

53

54

55

##########################################################################################################################################

56

### Run the program on an endless loop

57

##########################################################################################################################################

58

while(){

59

#Open the database connection

60

my $dsn = "DBI:mysql:$db_name:$db_host:$db_port";

61

my $dbh = DBI->connect($dsn,$db_user,"$db_pass");

62

63

#Get the list of locations to monitor

64

#------------------------------------------------------------------------------------------------------

65

my $sql = qq~SELECT * FROM FILE_MONITOR_SETTINGS WHERE status='1'~;

66

my $sth = $dbh->prepare($sql);

67

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

68

my @locations;

69

while(my $data = $sth->fetchrow_hashref){

70

push(@locations, $data);

71

}

72

#------------------------------------------------------------------------------------------------------

73

74

#Loop through all of the locations

75

foreach my $location(@locations){

76

#Get a the last file times we logged

77

#------------------------------------------------------------------------------------------------------

78

open(R, "$monitor_file-$location->{file_monitor_list_id}");

79

my @file_info = <R>;

80

close R;

81

#------------------------------------------------------------------------------------------------------

82

83

#Get the last changed file times from our saved file where we store the files and times

84

#------------------------------------------------------------------------------------------------------

85

my %file_info;

86

foreach my $line(@file_info){

87

chomp($line);

88

my($file, $mtime, $ctime, $version_filename, $uid, $gid, $permissions) = split(/\:/, $line);

89

$file_info{$file} = {'file_change' => $mtime, 'status_change' => $ctime, 'version_filename' => $version_filename, 'uid' => $uid, 'gid' => $gid, 'permissions' => $permissions};

90

}

91

#------------------------------------------------------------------------------------------------------

92

93

#Get the folder locations to scan

94

#------------------------------------------------------------------------------------------------------

95

my @location_list = split(/\n/, $location->{scan_path});

96

97

#Cleanup the list of folder locations by removing comments and leading/trailing spaces

98

my @monitor_folders;

99

foreach my $item(@location_list){

100

$item =~ s/^[\s\t]+//; #Remove leading spaces/tabs

101

$item =~ s/[\s\t]+$//; #Remove trailing spaces/tabs

102

103

next if $item =~ m/^\#/; #Skip comment rows

104

next if $item !~ m/[\w\d\/]+/; #Skip blank lines

105

106

push(@monitor_folders, $item); #Save the good item

107

#print qq~SCAN: $item\n~;

108

}

109

#------------------------------------------------------------------------------------------------------

110

111

#Get the list of files and folders to ignore

112

#------------------------------------------------------------------------------------------------------

113

my @ignore_list = split(/\n/, $location->{ignore_list});

114

115

#Cleanup the list of folder locations by removing comments and leading/trailing spaces

116

my @ignore;

117

foreach my $item(@ignore_list){

118

$item =~ s/^[\s\t]+//; #Remove leading spaces/tabs

119

$item =~ s/[\s\t]+$//; #Remove trailing spaces/tabs

120

121

next if $item =~ m/^\#/; #Skip comment rows

122

next if $item !~ m/[\w\d\/]+/; #Skip blank lines

123

124

push(@ignore, $item); #Save the good item

125

#print qq~SKIP: $item\n~;

126

}

127

#------------------------------------------------------------------------------------------------------

128

129

my @new_file_info;

130

foreach my $file_path(@monitor_folders){

131

#Get a list of all files in the chosen directory

132

#------------------------------------------------------------------------------------------------------

133

my @files = `find $file_path`;

134

foreach my $file(@files){

135

chomp($file);

136

}

137

#------------------------------------------------------------------------------------------------------

138

139

#Loop through the files in the directory we are scanning, get the times and see if anything changed

140

#------------------------------------------------------------------------------------------------------

141

foreach my $file(@files){

142

next if -d "$file"; #Skip of this is a directory

143

144

my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks)= stat("$file");

145

my $permissions = sprintf "%04o", $mode &07777;

146

147

next if !$file;

148

next if $file eq '.';

149

next if $file eq '..';

150

151

my $skip;

152

foreach my $ignore(@ignore){

153

$skip = '1' if $file =~ m/^$ignore$/; #Skip files that match

154

$skip = '1' if $file =~ m/^$ignore.*?/; #Skip folders that match

155

last if $skip; #We found the file, stop searching

156

}

157

next if $skip; #Skip this and go to the next row

158

159

#Used to find deleted files, we track all of the files we find, then in the end check which files were not found

160

$file_info{$file}->{file_exists} = '1';

161

162

#If something changed then track it, or if a new file was found

163

if($mtime != $file_info{$file}->{file_change} || $ctime != $file_info{$file}->{status_change}){

164

my $new_file;

165

my $version_filename;

166

if($file_info{$file}->{version_filename}){

167

#Use the exiting version filename since we already have one saved for this file

168

$version_filename = $file_info{$file}->{version_filename};

169

}else{

170

#Create a new version filename since we don't already have one saved for this file

171

$version_filename = time() . rand(20) . '.dat';

172

173

$new_file = '1';

174

}

175

176

177

#Read the file contents

178

open(R, "$file");

179

my @file_contents = <R>;

180

close R;

181

182

#Escape all bad chars

183

my $file_contents;

184

foreach my $line(@file_contents){

185

$line = quotemeta("$line");

186

187

#Restore the HTML's % sign

188

$line =~ s/\\\%/%/g;

189

190

#Save the prepared line of data

191

$file_contents .= $line;

192

}

193

194

if($new_file){

195

#New file

196

my $ltime = localtime();

197

#print qq~NEW FILE: "$file" ---> $ltime\n~;

198

199

#Add the database record

200

my $sql = qq~INSERT INTO FILE_CHANGES SET file_changes_id=NULL, file_monitor_list_id='$location->{file_monitor_list_id}', file_name='$file', file_contents=COMPRESS('$file_contents'), status='new file', uid=':$uid', gid=':$gid', permissions=':$permissions', date_time=NOW()~;

201

my $sth = $dbh->prepare($sql);

202

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

203

204

#Update the main category we are monitoring with the datetime of this last change

205

#------------------------------------------------------------------------------------------------------

206

my $sql = qq~UPDATE FILE_MONITOR_SETTINGS SET datetime_last_change=NOW() WHERE file_monitor_list_id='$location->{file_monitor_list_id}' LIMIT 1~;

207

my $sth = $dbh->prepare($sql);

208

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

209

#------------------------------------------------------------------------------------------------------

210

}else{

211

#File changed

212

my $ltime = localtime();

213

#print qq~FILE MODIFIED: "$file" ---> $ltime\n~;

214

215

#Run the diff

216

my $diff_results;

217

if(-e "$version_archive$version_filename"){

218

#Run the diff command to do the file comparison

219

$diff_results = `diff -u '$version_archive$version_filename' '$file'`;

220

$diff_results = quotemeta("$diff_results");

221

222

#Restore the HTML's % sign

223

$diff_results =~ s/\\\%/%/g;

224

}

225

226

#Add the database record

227

my $sql = qq~INSERT INTO FILE_CHANGES SET file_changes_id=NULL, file_monitor_list_id='$location->{file_monitor_list_id}', file_name='$file', diff=COMPRESS('$diff_results'), file_contents=COMPRESS('$file_contents'), uid='$file_info{$file}->{uid}:$uid', gid='$file_info{$file}->{gid}:$gid', permissions='$file_info{$file}->{permissions}:$permissions', status='file modified', date_time=NOW()~;

228

my $sth = $dbh->prepare($sql);

229

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

230

231

#Update the main category we are monitoring with the datetime of this last change

232

#------------------------------------------------------------------------------------------------------

233

my $sql = qq~UPDATE FILE_MONITOR_SETTINGS SET datetime_last_change=NOW() WHERE file_monitor_list_id='$location->{file_monitor_list_id}' LIMIT 1~;

234

my $sth = $dbh->prepare($sql);

235

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

236

#------------------------------------------------------------------------------------------------------

237

}

238

239

#Add the new file or update the timestamps of the old file, this handles both scenerios

240

$file_info{$file} = {'file_change' => $mtime, 'status_change' => $ctime, 'version_filename' => $version_filename, 'file_exists' => '1', 'uid' => $uid, 'gid' => $gid, 'permissions' => $permissions};

241

242

`cp '$file' '$version_archive$version_filename'`;

243

}

244

}

245

#------------------------------------------------------------------------------------------------------

246

}

247

248

249

#Update the file monitor log, also remove deleted files and marke them in the DB

250

#------------------------------------------------------------------------------------------------------

251

open(W, ">$monitor_file-$location->{file_monitor_list_id}");

252

253

foreach my $file(keys %file_info){

254

#Don't save deleted files, make sure the file exists

255

if($file_info{$file}->{file_exists}){

256

print W qq~$file:$file_info{$file}->{file_change}:$file_info{$file}->{status_change}:$file_info{$file}->{version_filename}:$file_info{$file}->{uid}:$file_info{$file}->{gid}:$file_info{$file}->{permissions}\n~;

257

}else{

258

#We don't want to mark files deleted if they were recently added to the ignore list so we need to look for those here and skip them if set to ignore instead of saying they were deleted

259

my $skip;

260

foreach my $ignore(@ignore){

261

$skip = '1' if $file =~ m/^$ignore$/; #Skip files that match

262

$skip = '1' if $file =~ m/^$ignore.*?/; #Skip folders that match

263

last if $skip; #We found the file, stop searching

264

}

265

next if $skip; #Skip this and go to the next row

266

267

#File not found, this file has been deleted

268

my $ltime = localtime();

269

#print qq~FILE DELETED: "$file" ---> $ltime\n~;

270

271

#Add the database record

272

my $sql = qq~INSERT INTO FILE_CHANGES SET file_changes_id=NULL, file_monitor_list_id='$location->{file_monitor_list_id}', file_name='$file', status='file deleted', uid='$file_info{$file}->{uid}:', gid='$file_info{$file}->{gid}:', permissions='$file_info{$file}->{permissions}:', date_time=NOW()~;

273

my $sth = $dbh->prepare($sql);

274

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

275

276

#Update the main category we are monitoring with the datetime of this last change

277

#------------------------------------------------------------------------------------------------------

278

my $sql = qq~UPDATE FILE_MONITOR_SETTINGS SET datetime_last_change=NOW() WHERE file_monitor_list_id='$location->{file_monitor_list_id}' LIMIT 1~;

279

my $sth = $dbh->prepare($sql);

280

$sth->execute || error("ERROR: line nunber:" . __LINE__ . " could not execute sql:$sql", 'nospaces');

281

#------------------------------------------------------------------------------------------------------

282

}

283

}

284

285

close W;

286

#------------------------------------------------------------------------------------------------------

287

}

288

289

#Close the database connection

290

$dbh->disconnect;

291

292

sleep($pause_seconds);

293

}

294

295

sub error{

296

open(A, ">>$error_file");

297

print A $_ . "\n";

298

close A;

299

}

300

301

302

#NOTES

303

#------------------------------------------------------------------------------------------------------

304

#ssh shawn2@98.7.189.34 -p22 "find /datastore/STG_SmartBOM/WebRoot/shawn_V2.0/cgi/SYSTEM_MONITOR/"

305

#Cat any files that are different on the remote server, the -t requests the user to type in their password, this works but I need to avoid typing passwords, what about SSH keys

306

#ssh -t shawn2@98.7.189.34 -p22 "sudo cat /datastore/STG_SmartBOM/WebRoot/shawn_V2.0/cgi/SYSTEM_MONITOR/db_fails.cgi"

307

#Mounting folders inside of the main one I am running in would allow me to scan those folders

308

#Allow Perl to run with CGI.pm from outside of the Docker containers

309

310

#Load the area with AJAX when clicked to view, different subroutine that does not use the Template.pm code will print it using AJAX so it doesn't need escaped

311

#The dropdown for selecting the day jumped ahead at 6pm to the next day, need to check how it is getting the date fields date/time and make sure it is adjusted to match the others.

312

#The "Last Processed" date on this page is from the schema monitor, this needs to be changed to the file monitor date/time for all file monitor pages

313

#------------------------------------------------------------------------------------------------------

Diff -- /var/www/vhosts/3dshawn.com/site1/_file_monitor.cgi

/var/www/vhosts/3dshawn.com/site1/_file_monitor.cgi