Diff -- /var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

O Operator
Diff

/var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

modified on local at 2026-07-12 00:02:10

Added
+18
lines
Removed
-2
lines
Context
436
unchanged
Blobs
from ee7754b6fe45
to bf5b44533145
Restore this content →
Unified diff
Naive LCS-based line diff. Additions in emerald, deletions in rose. Both sides decompressed live from BLOB_STORE.
11#!/usr/bin/perl
22#======================================================================
33# DriftSense -- SSH agent scanner (Stealth Flavor 2)
44#
55# Cron entry point (NOT a CGI). For each active SERVERS row with
66# kind='ssh_agent', SSHes into the monitored host using the configured
77# restricted key and walks the paths in that server's FILE_MONITOR_SETTINGS.
88#
99# Monitored host runs zero DriftSense code -- only a single command-
1010# restricted authorized_keys entry. The wrapper on the remote side
1111# only allows 3 commands:
1212#
1313# drift-find <path> -> path\tsize\tmtime\n per file
1414# drift-hash <files> -> sha256sum output
1515# drift-read <file> -> raw bytes
1616#
1717# We drive it via SSH_ORIGINAL_COMMAND. If the wrapper isn't installed,
1818# we fall back to plain `find`, `sha256sum`, `cat` on the assumption
1919# that the SSH user has a normal shell (test-mode).
2020#
2121# Config: /etc/drift_sense/drift_sense.conf (via MODS::Config).
2222# Log: /var/log/drift_sense/agent_scan.log (via cron redirect).
2323#======================================================================
2424use strict;
2525use warnings;
2626use lib '/var/www/vhosts/3dshawn.com/site1';
2727use POSIX ();
2828use Digest::SHA qw(sha256_hex);
2929use Compress::Zlib qw(compress);
3030use IPC::Open3;
3131use Symbol 'gensym';
3232use MODS::Config;
3333use MODS::DBConnect;
3434use MODS::Crypto;
3535use Fcntl ();
3636
3737$| = 1;
3838my $ts = POSIX::strftime('%Y-%m-%d %H:%M:%S', localtime);
3939
4040my $cfg = MODS::Config->new;
4141my $db = MODS::DBConnect->new;
4242my $dbh = $db->db_connect or die "[$ts] cannot connect to DriftSense storage DB\n";
4343
4444my $default_max_size = int($cfg->settings('max_file_size_bytes') || 10_485_760);
4545my $DEFAULT_TIMEOUT = 30;
46
47# ---- Global scan settings (SCAN_GLOBALS singleton) -----------------
48my $g_row = $db->db_readwrite($dbh, q~
49 SELECT global_ignore_list, global_file_type_filter, global_max_file_size_bytes
50 FROM SCAN_GLOBALS WHERE id = 1 LIMIT 1
51~, __FILE__, __LINE__);
52my $global_ignore = ($g_row && $g_row->{global_ignore_list}) // '';
53my $global_ftypes = ($g_row && $g_row->{global_file_type_filter}) // '';
54my $global_max_size = ($g_row && $g_row->{global_max_file_size_bytes}) || 0;
55$default_max_size = $global_max_size if $global_max_size;
4656
4757# ---- Fetch ssh_agent servers ----------------------------------------
4858my @servers = $db->db_readwrite_multiple($dbh, q~
4959 SELECT server_id, server_name, ssh_host, ssh_user, ssh_port,
5060 ssh_key_path, ssh_options, ssh_key_blob, ssh_key_enc
5161 FROM SERVERS
5262 WHERE kind = 'ssh_agent'
5363 AND status = 'active'
5464~, __FILE__, __LINE__);
5565
5666unless (@servers) {
5767 print "[$ts] no active ssh_agent servers, exiting\n";
5868 $db->db_disconnect($dbh);
5969 exit 0;
6070}
6171
6272my $total_added = 0;
6373my $total_modified = 0;
6474my $total_deleted = 0;
6575
6676foreach my $server (@servers) {
6777 my $sid = $server->{server_id};
6878 my $host = $server->{ssh_host} || next;
6979 my $user = $server->{ssh_user} || 'root';
7080 my $port = $server->{ssh_port} || 22;
7181 my $opts = $server->{ssh_options} || '';
7282
7383 # Resolve key: prefer encrypted blob (decrypt to a mode-600 temp
7484 # file for the SSH call), else fall back to on-disk ssh_key_path.
7585 my $key_path = $server->{ssh_key_path} || '';
7686 my $tmp_key;
7787 if ($server->{ssh_key_blob}) {
7888 my $pem = eval { MODS::Crypto::decrypt($server->{ssh_key_blob}) };
7989 if ($pem) {
8090 $tmp_key = _materialize_key($pem);
8191 $key_path = $tmp_key if $tmp_key;
8292 } else {
8393 print "[$ts] server $server->{server_name}: key decrypt failed ($@)\n";
8494 }
8595 }
8696
8797 my @ssh_base = _ssh_argv($user, $host, $port, $key_path, $opts);
8898
8999 # Fetch this server's active file monitors
90100 my @scans = $db->db_readwrite_multiple($dbh, qq~
91101 SELECT file_monitor_list_id, scan_path, ignore_list, file_type_filter,
92102 max_file_size_bytes, scan_name
93103 FROM FILE_MONITOR_SETTINGS
94104 WHERE status = 1
95105 AND server_id = $sid
96106 ~, __FILE__, __LINE__);
97107
98108 unless (@scans) {
99109 print "[$ts] server '$server->{server_name}' has no active monitors\n";
100110 next;
101111 }
102112
103113 my $server_err;
104114
105115 foreach my $scan (@scans) {
106116 my $path = $scan->{scan_path} or next;
107117 my $max_size = $scan->{max_file_size_bytes} || $default_max_size;
108118
109 my %ignore_pats = _build_ignore_patterns($scan->{ignore_list});
110 my %type_filter = _build_type_filter($scan->{file_type_filter});
119 # UNION global + per-monitor ignore lists
120 my $combined_ignore = join(',', grep { length } ($global_ignore, ($scan->{ignore_list} // '')));
121 my %ignore_pats = _build_ignore_patterns($combined_ignore);
122 # Per-monitor filter wins if set; else global
123 my $eff_ftypes = length($scan->{file_type_filter} // '')
124 ? $scan->{file_type_filter}
125 : $global_ftypes;
126 my %type_filter = _build_type_filter($eff_ftypes);
111127
112128 # ---- Remote listing -----------------------------------------
113129 my $remote_cmd = qq{drift-find $path};
114130 my ($rc, $out, $err) = _ssh_run(\@ssh_base, $remote_cmd);
115131 if ($rc != 0) {
116132 # Fallback: no wrapper -> plain shell
117133 my $shell = qq{find } . _sh_quote($path) . qq{ -type f -not -path '*/.*' -printf '%p\\t%s\\t%T@\\n' 2>/dev/null};
118134 ($rc, $out, $err) = _ssh_run(\@ssh_base, $shell);
119135 }
120136 if ($rc != 0) {
121137 $server_err = "list failed: $err";
122138 print "[$ts] server '$server->{server_name}' scan '$scan->{scan_name}': $server_err\n";
123139 next;
124140 }
125141
126142 # Parse listing: path\tsize\tmtime_epoch
127143 my %remote;
128144 foreach my $line (split /\n/, $out) {
129145 chomp $line;
130146 next unless length $line;
131147 my ($p, $sz, $mt) = split /\t/, $line, 3;
132148 next unless defined $p && defined $sz && defined $mt;
133149 $sz = int($sz);
134150 $mt = int($mt); # printf gives float mtime, truncate
135151
136152 # Ignore-glob filter (local side)
137153 my $skip = 0;
138154 for my $pat (keys %ignore_pats) {
139155 my $re = $ignore_pats{$pat};
140156 if ($p =~ $re) { $skip = 1; last; }
141157 }
142158 next if $skip;
143159
144160 # File-type filter (local side)
145161 if (%type_filter) {
146162 my $ext_ok = 0;
147163 for my $e (keys %type_filter) {
148164 if (lc(substr($p, -length($e))) eq $e) { $ext_ok = 1; last; }
149165 }
150166 next unless $ext_ok;
151167 }
152168
153169 next if $sz > $max_size;
154170 $remote{$p} = { size => $sz, mtime => $mt };
155171 }
156172
157173 # ---- Previous state ----------------------------------------
158174 my $prev_sth = $dbh->prepare(qq{
159175 SELECT fc.file_name, fc.blob_sha, fc.status,
160176 UNIX_TIMESTAMP(fc.date_time) AS mt_epoch
161177 FROM FILE_CHANGES fc
162178 WHERE fc.file_monitor_list_id = $scan->{file_monitor_list_id}
163179 AND fc.server_id = $sid
164180 AND fc.file_changes_id IN (
165181 SELECT MAX(inner_fc.file_changes_id)
166182 FROM FILE_CHANGES inner_fc
167183 WHERE inner_fc.file_monitor_list_id = $scan->{file_monitor_list_id}
168184 AND inner_fc.server_id = $sid
169185 GROUP BY inner_fc.file_name
170186 )
171187 });
172188 my %prev;
173189 if ($prev_sth && $prev_sth->execute) {
174190 while (my $r = $prev_sth->fetchrow_hashref) {
175191 $prev{$r->{file_name}} = $r;
176192 }
177193 $prev_sth->finish;
178194 }
179195
180196 # ---- Decide who needs a fresh hash --------------------------
181197 my @needs_hash;
182198 foreach my $p (keys %remote) {
183199 my $prev_row = $prev{$p};
184200 if (!$prev_row || ($prev_row->{status} // '') eq 'deleted') {
185201 push @needs_hash, $p;
186202 next;
187203 }
188204 if (($prev_row->{mt_epoch} // 0) != $remote{$p}{mtime}) {
189205 push @needs_hash, $p;
190206 }
191207 }
192208
193209 # ---- Batch hash over SSH -----------------------------------
194210 # `drift-hash <files>` returns sha256sum output: "<sha> <path>"
195211 my %fresh_sha;
196212 if (@needs_hash) {
197213 my $q_list = join(' ', map { _sh_quote($_) } @needs_hash);
198214 my $cmd = "drift-hash $q_list";
199215 my ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
200216 if ($h_rc != 0) {
201217 # Fallback: plain sha256sum
202218 $cmd = "sha256sum $q_list 2>/dev/null";
203219 ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
204220 }
205221 if ($h_rc == 0) {
206222 foreach my $line (split /\n/, $h_out) {
207223 chomp $line;
208224 if ($line =~ /^([0-9a-f]{64})\s+(.+)$/) {
209225 $fresh_sha{$2} = $1;
210226 }
211227 }
212228 }
213229 }
214230
215231 # ---- Process each remote file ------------------------------
216232 foreach my $p (keys %remote) {
217233 my $sha = $fresh_sha{$p};
218234 my $prev_row = $prev{$p};
219235
220236 # If we didn't rehash this file, previous SHA still valid
221237 if (!defined $sha && $prev_row && ($prev_row->{status} // '') ne 'deleted') {
222238 # Nothing changed since last tick -- move on
223239 next;
224240 }
225241
226242 # Content-addressable dedup: if we already have this SHA in
227243 # BLOB_STORE, don't re-fetch content over SSH.
228244 my $have = $db->db_readwrite($dbh,
229245 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
230246 __FILE__, __LINE__);
231247
232248 my $status;
233249 if (!$prev_row) {
234250 $status = 'added';
235251 } else {
236252 $status = 'modified';
237253 if ($prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
238254 # Same bytes, just an mtime touch -- flag ts-only
239255 $status = 'modified';
240256 }
241257 }
242258
243259 # Fetch + store blob only if we don't already have it
244260 if (!($have && $have->{blob_sha})) {
245261 my $cmd = 'drift-read ' . _sh_quote($p);
246262 my ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
247263 if ($c_rc != 0) {
248264 $cmd = 'cat ' . _sh_quote($p);
249265 ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
250266 }
251267 if ($c_rc != 0) {
252268 print "[$ts] fetch $p on $server->{server_name}: $c_err\n";
253269 next;
254270 }
255271 # Verify the fetched content matches the reported hash
256272 my $verify_sha = sha256_hex($c_out);
257273 if ($verify_sha ne $sha) {
258274 print "[$ts] $p: hash mismatch (list=$sha vs read=$verify_sha), using read\n";
259275 $sha = $verify_sha;
260276 }
261277 _store_blob($db, $dbh, $sha, $c_out);
262278 }
263279
264280 # Determine timestamp-only via prev SHA compare
265281 my $is_ts_only = 0;
266282 if ($prev_row && $prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
267283 $is_ts_only = 1;
268284 }
269285
270286 # Insert FILE_CHANGES row keyed to this server
271287 my $q_file = $dbh->quote($p);
272288 my $q_sha = $dbh->quote($sha);
273289 my $q_stat = $dbh->quote($status);
274290 my $mt = $remote{$p}{mtime};
275291 $db->db_readwrite($dbh, qq{
276292 INSERT INTO FILE_CHANGES
277293 (file_monitor_list_id, server_id, container_target_id,
278294 file_name, blob_sha, is_ts_only, date_time, status)
279295 VALUES
280296 ($scan->{file_monitor_list_id}, $sid, NULL,
281297 $q_file, $q_sha, $is_ts_only,
282298 FROM_UNIXTIME($mt), $q_stat)
283299 }, __FILE__, __LINE__);
284300
285301 if ($is_ts_only) { }
286302 elsif ($status eq 'added') { $total_added++; }
287303 else { $total_modified++; }
288304 }
289305
290306 # ---- Delete detection --------------------------------------
291307 foreach my $p (keys %prev) {
292308 next if $remote{$p};
293309 next if ($prev{$p}{status} // '') eq 'deleted';
294310 my $q_file = $dbh->quote($p);
295311 $db->db_readwrite($dbh, qq{
296312 INSERT INTO FILE_CHANGES
297313 (file_monitor_list_id, server_id, file_name, status, date_time)
298314 VALUES
299315 ($scan->{file_monitor_list_id}, $sid, $q_file, 'deleted', NOW())
300316 }, __FILE__, __LINE__);
301317 $total_deleted++;
302318 }
303319
304320 $db->db_readwrite($dbh, qq{
305321 UPDATE FILE_MONITOR_SETTINGS
306322 SET last_scanned = NOW()
307323 WHERE file_monitor_list_id = $scan->{file_monitor_list_id}
308324 }, __FILE__, __LINE__);
309325 }
310326
311327 # Server-level bookkeeping
312328 my $q_err = $dbh->quote($server_err || '');
313329 $db->db_readwrite($dbh, qq{
314330 UPDATE SERVERS
315331 SET last_agent_scan_at = NOW(),
316332 last_agent_error = $q_err,
317333 last_heartbeat_at = NOW()
318334 WHERE server_id = $sid
319335 }, __FILE__, __LINE__);
320336
321337 # Clean up decrypted-key temp file
322338 unlink $tmp_key if $tmp_key && $tmp_key =~ m!^/tmp/\.ds_agent_key_!;
323339}
324340
325341$db->db_disconnect($dbh);
326342print "[$ts] agent scan complete: +$total_added added, ~$total_modified modified, -$total_deleted deleted\n"
327343 if ($total_added + $total_modified + $total_deleted) > 0;
328344exit 0;
329345
330346#---------------------------------------------------------------------
331347sub _ssh_argv {
332348 my ($user, $host, $port, $key, $opts) = @_;
333349 my @a = ('/usr/bin/ssh',
334350 '-o', 'BatchMode=yes',
335351 '-o', 'StrictHostKeyChecking=no',
336352 '-o', 'UserKnownHostsFile=/dev/null',
337353 '-o', 'LogLevel=ERROR',
338354 '-o', 'ConnectTimeout=8',
339355 '-p', $port);
340356 if ($key && -r $key) {
341357 push @a, '-i', $key;
342358 }
343359 if ($opts) {
344360 # Space-separated -o key=value pairs
345361 foreach my $o (split /\s+/, $opts) {
346362 push @a, '-o', $o if length $o;
347363 }
348364 }
349365 push @a, "$user\@$host";
350366 return @a;
351367}
352368
353369sub _ssh_run {
354370 my ($base, $cmd) = @_;
355371 my @argv = (@$base, $cmd);
356372 my ($wtr, $rdr, $err_fh);
357373 $err_fh = gensym;
358374 my $pid = eval { open3($wtr, $rdr, $err_fh, @argv) };
359375 return (-1, '', $@) if $@ || !$pid;
360376 close $wtr;
361377 binmode $rdr;
362378 my $out = do { local $/; <$rdr> // '' };
363379 my $err = do { local $/; <$err_fh> // '' };
364380 close $rdr; close $err_fh;
365381 waitpid $pid, 0;
366382 my $rc = $? >> 8;
367383 return ($rc, $out, $err);
368384}
369385
370386sub _materialize_key {
371387 my ($pem) = @_;
372388 return undef unless defined $pem && length $pem;
373389 my $t = POSIX::strftime('%s', localtime);
374390 my $path = "/tmp/.ds_agent_key_${t}_$$_" . int(rand(1_000_000));
375391 sysopen(my $fh, $path,
376392 Fcntl::O_WRONLY() | Fcntl::O_CREAT() | Fcntl::O_EXCL(),
377393 0600) or return undef;
378394 print $fh $pem;
379395 close $fh;
380396 return $path;
381397}
382398
383399sub _sh_quote {
384400 my $s = shift; $s //= '';
385401 # Single-quote and escape internal single quotes
386402 $s =~ s/'/'\\''/g;
387403 return "'$s'";
388404}
389405
390406sub _build_ignore_patterns {
391407 my $list = shift;
392408 my %out;
393409 return %out unless $list;
394410 for my $ig (split /[,\n]+/, $list) {
395411 $ig =~ s/^\s+|\s+$//g;
396412 next unless length $ig;
397413 my $re = quotemeta $ig;
398414 $re =~ s/\\\*/.*/g;
399415 $re =~ s/\\\?/./g;
400416 $out{$ig} = qr/$re/;
401417 }
402418 return %out;
403419}
404420
405421sub _build_type_filter {
406422 my $list = shift;
407423 my %out;
408424 return %out unless $list && length $list;
409425 for my $ext (split /,/, $list) {
410426 $ext =~ s/^\s+|\s+$//g;
411427 next unless length $ext;
412428 $ext = ".$ext" unless $ext =~ /^\./;
413429 $out{lc $ext} = 1;
414430 }
415431 return %out;
416432}
417433
418434sub _store_blob {
419435 my ($db, $dbh, $sha, $content) = @_;
420436 my $existing = $db->db_readwrite($dbh,
421437 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
422438 __FILE__, __LINE__);
423439 if ($existing && $existing->{blob_sha}) {
424440 $db->db_readwrite($dbh,
425441 "UPDATE BLOB_STORE SET ref_count = ref_count + 1 WHERE blob_sha = " . $dbh->quote($sha),
426442 __FILE__, __LINE__);
427443 return;
428444 }
429445 my $gz = compress($content) // '';
430446 my $sth = $dbh->prepare(qq{
431447 INSERT INTO BLOB_STORE (blob_sha, content_gz, size_uncompressed, size_compressed, ref_count, first_seen_at)
432448 VALUES (?, ?, ?, ?, 1, NOW())
433449 });
434450 if ($sth) {
435451 $sth->execute($sha, $gz, length($content), length($gz));
436452 $sth->finish;
437453 }
438454}