Diff -- /var/www/vhosts/3dshawn.com/site1/_file_scan.pl

O Operator
Diff

/var/www/vhosts/3dshawn.com/site1/_file_scan.pl

modified on local at 2026-07-12 00:02:09

Added
+23
lines
Removed
-4
lines
Context
264
unchanged
Blobs
from 2b862b37ab7e
to 41cc7dc8579b
Restore this content →
Unified diff
Naive LCS-based line diff. Additions in emerald, deletions in rose. Both sides decompressed live from BLOB_STORE.
11#!/usr/bin/perl
22#======================================================================
33# DriftSense -- filesystem scanner
44#
55# Cron entry point (NOT a CGI). Reads FILE_MONITOR_SETTINGS, walks each
66# configured path, and captures every changed file since the previous
77# tick. Optimizations:
88#
99# * mtime-first check -- cheap stat() every file, hash only when
1010# mtime differs vs. the last captured version. Turns a 10 GB
1111# codebase scan from minutes into seconds.
1212# * Content-addressable BLOB_STORE dedup -- if the file's content
1313# bounces back to a previous state, zero new bytes stored.
1414# * gzip compression on stored snapshots.
1515# * Timestamp-only change suppression (per Shawn's own note file)
1616# -- files whose only change is mtime (content-identical) get
1717# flagged is_ts_only=1 so the UI can filter noise.
1818# * Per-scan max_file_size and file_type_filter honored.
1919#
2020# Config: /etc/drift_sense/drift_sense.conf (via MODS::Config).
2121# Log: /var/log/drift_sense/file_scan.log (via cron redirect).
2222#======================================================================
2323use strict;
2424use warnings;
2525use lib '/var/www/vhosts/3dshawn.com/site1';
2626use POSIX ();
2727use Digest::SHA qw(sha256_hex);
2828use Compress::Zlib qw(compress);
2929use File::Find ();
3030use MODS::Config;
3131use MODS::DBConnect;
3232
3333$| = 1;
3434my $ts = POSIX::strftime('%Y-%m-%d %H:%M:%S', localtime);
3535
3636my $cfg = MODS::Config->new;
3737my $db = MODS::DBConnect->new;
3838my $dbh = $db->db_connect or die "[$ts] cannot connect to DriftSense storage DB\n";
3939
4040# ---- Config: global max-file-size default ----
4141my $default_max_size = int($cfg->settings('max_file_size_bytes') || 10_485_760);
42
43# ---- Global scan settings (SCAN_GLOBALS singleton) -----------------
44# Applied UNIVERSALLY across every monitor so operators can set
45# "always ignore .swp, node_modules/, etc." in one place.
46my $g_row = $db->db_readwrite($dbh, q~
47 SELECT global_ignore_list, global_file_type_filter, global_max_file_size_bytes
48 FROM SCAN_GLOBALS WHERE id = 1 LIMIT 1
49~, __FILE__, __LINE__);
50my $global_ignore = ($g_row && $g_row->{global_ignore_list}) // '';
51my $global_ftypes = ($g_row && $g_row->{global_file_type_filter}) // '';
52my $global_max_size = ($g_row && $g_row->{global_max_file_size_bytes}) || 0;
53$default_max_size = $global_max_size if $global_max_size;
4254
4355# ---- Fetch active file monitors ----
4456my @scans = $db->db_readwrite_multiple($dbh, q~
4557 SELECT file_monitor_list_id, scan_path, ignore_list, file_type_filter,
4658 max_file_size_bytes, server_id, container_target_id, scan_name
4759 FROM FILE_MONITOR_SETTINGS
4860 WHERE status = 1
4961~, __FILE__, __LINE__);
5062
5163unless (@scans) {
5264 print "[$ts] no active file monitors, exiting\n";
5365 exit 0;
5466}
5567
5668my $total_added = 0;
5769my $total_modified = 0;
5870my $total_deleted = 0;
5971my $total_ts_only = 0;
6072
6173foreach my $scan (@scans) {
6274 my $path = $scan->{scan_path} or next;
6375 unless (-d $path) {
6476 print "[$ts] scan #$scan->{file_monitor_list_id} '$scan->{scan_name}': path missing ($path)\n";
6577 next;
6678 }
6779
80 # ---- Ignore patterns: UNION of global + per-monitor ------------
6881 my %ignore_pats;
69 if ($scan->{ignore_list}) {
70 for my $ig (split /[,\n]+/, $scan->{ignore_list}) {
82 for my $src ($global_ignore, ($scan->{ignore_list} // '')) {
83 next unless length $src;
84 for my $ig (split /[,\n]+/, $src) {
7185 $ig =~ s/^\s+|\s+$//g;
7286 $ignore_pats{$ig} = _pattern_to_regex($ig) if length $ig;
7387 }
7488 }
7589
90 # ---- File-type filter: per-monitor wins if set, else global ---
7691 my %type_filter;
77 if ($scan->{file_type_filter} && length $scan->{file_type_filter}) {
78 for my $ext (split /,/, $scan->{file_type_filter}) {
92 my $eff_ftypes = length($scan->{file_type_filter} // '')
93 ? $scan->{file_type_filter}
94 : $global_ftypes;
95 if ($eff_ftypes && length $eff_ftypes) {
96 for my $ext (split /,/, $eff_ftypes) {
7997 $ext =~ s/^\s+|\s+$//g;
98 next unless length $ext;
8099 $ext = ".$ext" unless $ext =~ /^\./;
81100 $type_filter{lc $ext} = 1;
82101 }
83102 }
84103
85104 my $max_size = $scan->{max_file_size_bytes} || $default_max_size;
86105
87106 # ---- Snapshot of what we captured in the previous tick, keyed by
88107 # file_name, for delete-detection + mtime-baseline lookup.
89108 my $prev_sth = $dbh->prepare(qq~
90109 SELECT fc.file_name, fc.blob_sha, fc.status, fc.date_time
91110 FROM FILE_CHANGES fc
92111 WHERE fc.file_monitor_list_id = $scan->{file_monitor_list_id}
93112 AND fc.file_changes_id IN (
94113 SELECT MAX(inner_fc.file_changes_id)
95114 FROM FILE_CHANGES inner_fc
96115 WHERE inner_fc.file_monitor_list_id = $scan->{file_monitor_list_id}
97116 GROUP BY inner_fc.file_name
98117 )
99118 ~);
100119 my %prev;
101120 if ($prev_sth && $prev_sth->execute) {
102121 while (my $r = $prev_sth->fetchrow_hashref) {
103122 $prev{$r->{file_name}} = $r;
104123 }
105124 $prev_sth->finish;
106125 }
107126
108127 # ---- Walk + capture ----
109128 my %seen;
110129 File::Find::find({
111130 no_chdir => 1,
112131 wanted => sub {
113132 my $f = $File::Find::name;
114133 return if -d $f;
115134 return unless -f $f;
116135
117136 # Ignore patterns
118137 for my $pat (keys %ignore_pats) {
119138 my $re = $ignore_pats{$pat};
120139 if ($f =~ $re) { return; }
121140 }
122141
123142 # File type filter
124143 if (%type_filter) {
125144 my $ext_ok = 0;
126145 for my $e (keys %type_filter) {
127146 if (lc(substr($f, -length($e))) eq $e) { $ext_ok = 1; last; }
128147 }
129148 return unless $ext_ok;
130149 }
131150
132151 my @st = stat $f;
133152 return unless @st;
134153 return if $st[7] > $max_size; # skip huge files
135154
136155 $seen{$f} = 1;
137156 my $mtime = $st[9];
138157 my $prev_row = $prev{$f};
139158
140159 # mtime-first optimization: skip hashing if mtime hasn't advanced
141160 # (very common case for unchanged files).
142161 if ($prev_row && $prev_row->{status} ne 'deleted') {
143162 my $prev_mtime = _parse_mysql_datetime($prev_row->{date_time});
144163 return if $prev_mtime && $prev_mtime == $mtime;
145164 }
146165
147166 # Read + hash
148167 my $content = _slurp($f);
149168 return unless defined $content;
150169 my $sha = sha256_hex($content);
151170
152171 # Same content as previous capture? timestamp-only change.
153172 my $is_ts_only = 0;
154173 my $status = 'added';
155174 if ($prev_row && $prev_row->{blob_sha}) {
156175 if ($prev_row->{blob_sha} eq $sha) {
157176 $is_ts_only = 1;
158177 $status = 'modified'; # mtime touched but bytes identical
159178 } else {
160179 $status = 'modified';
161180 }
162181 }
163182
164183 # Persist blob (content-addressable, dedup)
165184 _store_blob($db, $dbh, $sha, $content);
166185
167186 # Insert change row
168187 my $q_file = $dbh->quote($f);
169188 my $q_sha = $dbh->quote($sha);
170189 my $q_stat = $dbh->quote($status);
171190 my $uid = $st[4]; my $gid = $st[5];
172191 my $perm = sprintf('%04o', $st[2] & 07777);
173192 $db->db_readwrite($dbh, qq~
174193 INSERT INTO FILE_CHANGES
175194 (file_monitor_list_id, server_id, container_target_id,
176195 file_name, blob_sha, is_ts_only, date_time, status,
177196 uid, gid, permissions)
178197 VALUES
179198 ($scan->{file_monitor_list_id}, $scan->{server_id},
180199 ~ . ($scan->{container_target_id} ? $scan->{container_target_id} : 'NULL') . qq~,
181200 $q_file, $q_sha, $is_ts_only,
182201 FROM_UNIXTIME($mtime), $q_stat,
183202 '$uid', '$gid', '$perm')
184203 ~, __FILE__, __LINE__);
185204
186205 if ($is_ts_only) { $total_ts_only++; }
187206 elsif ($status eq 'added') { $total_added++; }
188207 else { $total_modified++; }
189208 },
190209 }, $path);
191210
192211 # ---- Delete detection ----
193212 foreach my $f (keys %prev) {
194213 next if $seen{$f};
195214 next if $prev{$f}->{status} eq 'deleted'; # already marked deleted
196215 my $q_file = $dbh->quote($f);
197216 $db->db_readwrite($dbh, qq~
198217 INSERT INTO FILE_CHANGES
199218 (file_monitor_list_id, server_id, file_name, status, date_time)
200219 VALUES
201220 ($scan->{file_monitor_list_id}, $scan->{server_id},
202221 $q_file, 'deleted', NOW())
203222 ~, __FILE__, __LINE__);
204223 $total_deleted++;
205224 }
206225
207226 # Update last_scanned stamp
208227 $db->db_readwrite($dbh, qq~
209228 UPDATE FILE_MONITOR_SETTINGS
210229 SET last_scanned = NOW()
211230 WHERE file_monitor_list_id = $scan->{file_monitor_list_id}
212231 ~, __FILE__, __LINE__);
213232}
214233
215234$db->db_disconnect($dbh);
216235
217236print "[$ts] file scan complete: +$total_added added, ~$total_modified modified, "
218237 . "-$total_deleted deleted, $total_ts_only ts-only\n"
219238 if ($total_added + $total_modified + $total_deleted + $total_ts_only) > 0;
220239
221240exit 0;
222241
223242#---------------------------------------------------------------------
224243sub _slurp {
225244 my $path = shift;
226245 open(my $fh, '<:raw', $path) or return undef;
227246 local $/; my $c = <$fh>; close $fh;
228247 return $c;
229248}
230249
231250sub _pattern_to_regex {
232251 my $pat = shift;
233252 # Simple glob-ish -> regex conversion
234253 my $re = quotemeta $pat;
235254 $re =~ s/\\\*/.*/g;
236255 $re =~ s/\\\?/./g;
237256 return qr/$re/;
238257}
239258
240259sub _parse_mysql_datetime {
241260 my $dt = shift;
242261 return 0 unless $dt && $dt =~ /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})$/;
243262 require Time::Local;
244263 return eval { Time::Local::timelocal($6, $5, $4, $3, $2 - 1, $1 - 1900) } || 0;
245264}
246265
247266#---------------------------------------------------------------------
248267sub _store_blob {
249268 my ($db, $dbh, $sha, $content) = @_;
250269 my $existing = $db->db_readwrite($dbh,
251270 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
252271 __FILE__, __LINE__);
253272 if ($existing && $existing->{blob_sha}) {
254273 $db->db_readwrite($dbh,
255274 "UPDATE BLOB_STORE SET ref_count = ref_count + 1 WHERE blob_sha = " . $dbh->quote($sha),
256275 __FILE__, __LINE__);
257276 return;
258277 }
259278 my $gz = compress($content) // '';
260279 my $sth = $dbh->prepare(qq~
261280 INSERT INTO BLOB_STORE (blob_sha, content_gz, size_uncompressed, size_compressed, ref_count, first_seen_at)
262281 VALUES (?, ?, ?, ?, 1, NOW())
263282 ~);
264283 if ($sth) {
265284 $sth->execute($sha, $gz, length($content), length($gz));
266285 $sth->finish;
267286 }
268287}