Diff -- /var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

O Operator
Diff

/var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

modified on local at 2026-07-11 23:15:11

Added
+34
lines
Removed
-3
lines
Context
404
unchanged
Blobs
from e833d76d304c
to ee7754b6fe45
Restore this content →
Unified diff
Naive LCS-based line diff. Additions in emerald, deletions in rose. Both sides decompressed live from BLOB_STORE.
11#!/usr/bin/perl
22#======================================================================
33# DriftSense -- SSH agent scanner (Stealth Flavor 2)
44#
55# Cron entry point (NOT a CGI). For each active SERVERS row with
66# kind='ssh_agent', SSHes into the monitored host using the configured
77# restricted key and walks the paths in that server's FILE_MONITOR_SETTINGS.
88#
99# Monitored host runs zero DriftSense code -- only a single command-
1010# restricted authorized_keys entry. The wrapper on the remote side
1111# only allows 3 commands:
1212#
1313# drift-find <path> -> path\tsize\tmtime\n per file
1414# drift-hash <files> -> sha256sum output
1515# drift-read <file> -> raw bytes
1616#
1717# We drive it via SSH_ORIGINAL_COMMAND. If the wrapper isn't installed,
1818# we fall back to plain `find`, `sha256sum`, `cat` on the assumption
1919# that the SSH user has a normal shell (test-mode).
2020#
2121# Config: /etc/drift_sense/drift_sense.conf (via MODS::Config).
2222# Log: /var/log/drift_sense/agent_scan.log (via cron redirect).
2323#======================================================================
2424use strict;
2525use warnings;
2626use lib '/var/www/vhosts/3dshawn.com/site1';
2727use POSIX ();
2828use Digest::SHA qw(sha256_hex);
2929use Compress::Zlib qw(compress);
3030use IPC::Open3;
3131use Symbol 'gensym';
3232use MODS::Config;
3333use MODS::DBConnect;
34use MODS::Crypto;
35use Fcntl ();
3436
3537$| = 1;
3638my $ts = POSIX::strftime('%Y-%m-%d %H:%M:%S', localtime);
3739
3840my $cfg = MODS::Config->new;
3941my $db = MODS::DBConnect->new;
4042my $dbh = $db->db_connect or die "[$ts] cannot connect to DriftSense storage DB\n";
4143
4244my $default_max_size = int($cfg->settings('max_file_size_bytes') || 10_485_760);
4345my $DEFAULT_TIMEOUT = 30;
4446
4547# ---- Fetch ssh_agent servers ----------------------------------------
4648my @servers = $db->db_readwrite_multiple($dbh, q~
4749 SELECT server_id, server_name, ssh_host, ssh_user, ssh_port,
48 ssh_key_path, ssh_options
50 ssh_key_path, ssh_options, ssh_key_blob, ssh_key_enc
4951 FROM SERVERS
5052 WHERE kind = 'ssh_agent'
5153 AND status = 'active'
5254~, __FILE__, __LINE__);
5355
5456unless (@servers) {
5557 print "[$ts] no active ssh_agent servers, exiting\n";
5658 $db->db_disconnect($dbh);
5759 exit 0;
5860}
5961
6062my $total_added = 0;
6163my $total_modified = 0;
6264my $total_deleted = 0;
6365
6466foreach my $server (@servers) {
6567 my $sid = $server->{server_id};
6668 my $host = $server->{ssh_host} || next;
6769 my $user = $server->{ssh_user} || 'root';
6870 my $port = $server->{ssh_port} || 22;
69 my $key = $server->{ssh_key_path} || '';
7071 my $opts = $server->{ssh_options} || '';
7172
72 my @ssh_base = _ssh_argv($user, $host, $port, $key, $opts);
73 # Resolve key: prefer encrypted blob (decrypt to a mode-600 temp
74 # file for the SSH call), else fall back to on-disk ssh_key_path.
75 my $key_path = $server->{ssh_key_path} || '';
76 my $tmp_key;
77 if ($server->{ssh_key_blob}) {
78 my $pem = eval { MODS::Crypto::decrypt($server->{ssh_key_blob}) };
79 if ($pem) {
80 $tmp_key = _materialize_key($pem);
81 $key_path = $tmp_key if $tmp_key;
82 } else {
83 print "[$ts] server $server->{server_name}: key decrypt failed ($@)\n";
84 }
85 }
86
87 my @ssh_base = _ssh_argv($user, $host, $port, $key_path, $opts);
7388
7489 # Fetch this server's active file monitors
7590 my @scans = $db->db_readwrite_multiple($dbh, qq~
7691 SELECT file_monitor_list_id, scan_path, ignore_list, file_type_filter,
7792 max_file_size_bytes, scan_name
7893 FROM FILE_MONITOR_SETTINGS
7994 WHERE status = 1
8095 AND server_id = $sid
8196 ~, __FILE__, __LINE__);
8297
8398 unless (@scans) {
8499 print "[$ts] server '$server->{server_name}' has no active monitors\n";
85100 next;
86101 }
87102
88103 my $server_err;
89104
90105 foreach my $scan (@scans) {
91106 my $path = $scan->{scan_path} or next;
92107 my $max_size = $scan->{max_file_size_bytes} || $default_max_size;
93108
94109 my %ignore_pats = _build_ignore_patterns($scan->{ignore_list});
95110 my %type_filter = _build_type_filter($scan->{file_type_filter});
96111
97112 # ---- Remote listing -----------------------------------------
98113 my $remote_cmd = qq{drift-find $path};
99114 my ($rc, $out, $err) = _ssh_run(\@ssh_base, $remote_cmd);
100115 if ($rc != 0) {
101116 # Fallback: no wrapper -> plain shell
102117 my $shell = qq{find } . _sh_quote($path) . qq{ -type f -not -path '*/.*' -printf '%p\\t%s\\t%T@\\n' 2>/dev/null};
103118 ($rc, $out, $err) = _ssh_run(\@ssh_base, $shell);
104119 }
105120 if ($rc != 0) {
106121 $server_err = "list failed: $err";
107122 print "[$ts] server '$server->{server_name}' scan '$scan->{scan_name}': $server_err\n";
108123 next;
109124 }
110125
111126 # Parse listing: path\tsize\tmtime_epoch
112127 my %remote;
113128 foreach my $line (split /\n/, $out) {
114129 chomp $line;
115130 next unless length $line;
116131 my ($p, $sz, $mt) = split /\t/, $line, 3;
117132 next unless defined $p && defined $sz && defined $mt;
118133 $sz = int($sz);
119134 $mt = int($mt); # printf gives float mtime, truncate
120135
121136 # Ignore-glob filter (local side)
122137 my $skip = 0;
123138 for my $pat (keys %ignore_pats) {
124139 my $re = $ignore_pats{$pat};
125140 if ($p =~ $re) { $skip = 1; last; }
126141 }
127142 next if $skip;
128143
129144 # File-type filter (local side)
130145 if (%type_filter) {
131146 my $ext_ok = 0;
132147 for my $e (keys %type_filter) {
133148 if (lc(substr($p, -length($e))) eq $e) { $ext_ok = 1; last; }
134149 }
135150 next unless $ext_ok;
136151 }
137152
138153 next if $sz > $max_size;
139154 $remote{$p} = { size => $sz, mtime => $mt };
140155 }
141156
142157 # ---- Previous state ----------------------------------------
143158 my $prev_sth = $dbh->prepare(qq{
144159 SELECT fc.file_name, fc.blob_sha, fc.status,
145160 UNIX_TIMESTAMP(fc.date_time) AS mt_epoch
146161 FROM FILE_CHANGES fc
147162 WHERE fc.file_monitor_list_id = $scan->{file_monitor_list_id}
148163 AND fc.server_id = $sid
149164 AND fc.file_changes_id IN (
150165 SELECT MAX(inner_fc.file_changes_id)
151166 FROM FILE_CHANGES inner_fc
152167 WHERE inner_fc.file_monitor_list_id = $scan->{file_monitor_list_id}
153168 AND inner_fc.server_id = $sid
154169 GROUP BY inner_fc.file_name
155170 )
156171 });
157172 my %prev;
158173 if ($prev_sth && $prev_sth->execute) {
159174 while (my $r = $prev_sth->fetchrow_hashref) {
160175 $prev{$r->{file_name}} = $r;
161176 }
162177 $prev_sth->finish;
163178 }
164179
165180 # ---- Decide who needs a fresh hash --------------------------
166181 my @needs_hash;
167182 foreach my $p (keys %remote) {
168183 my $prev_row = $prev{$p};
169184 if (!$prev_row || ($prev_row->{status} // '') eq 'deleted') {
170185 push @needs_hash, $p;
171186 next;
172187 }
173188 if (($prev_row->{mt_epoch} // 0) != $remote{$p}{mtime}) {
174189 push @needs_hash, $p;
175190 }
176191 }
177192
178193 # ---- Batch hash over SSH -----------------------------------
179194 # `drift-hash <files>` returns sha256sum output: "<sha> <path>"
180195 my %fresh_sha;
181196 if (@needs_hash) {
182197 my $q_list = join(' ', map { _sh_quote($_) } @needs_hash);
183198 my $cmd = "drift-hash $q_list";
184199 my ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
185200 if ($h_rc != 0) {
186201 # Fallback: plain sha256sum
187202 $cmd = "sha256sum $q_list 2>/dev/null";
188203 ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
189204 }
190205 if ($h_rc == 0) {
191206 foreach my $line (split /\n/, $h_out) {
192207 chomp $line;
193208 if ($line =~ /^([0-9a-f]{64})\s+(.+)$/) {
194209 $fresh_sha{$2} = $1;
195210 }
196211 }
197212 }
198213 }
199214
200215 # ---- Process each remote file ------------------------------
201216 foreach my $p (keys %remote) {
202217 my $sha = $fresh_sha{$p};
203218 my $prev_row = $prev{$p};
204219
205220 # If we didn't rehash this file, previous SHA still valid
206221 if (!defined $sha && $prev_row && ($prev_row->{status} // '') ne 'deleted') {
207222 # Nothing changed since last tick -- move on
208223 next;
209224 }
210225
211226 # Content-addressable dedup: if we already have this SHA in
212227 # BLOB_STORE, don't re-fetch content over SSH.
213228 my $have = $db->db_readwrite($dbh,
214229 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
215230 __FILE__, __LINE__);
216231
217232 my $status;
218233 if (!$prev_row) {
219234 $status = 'added';
220235 } else {
221236 $status = 'modified';
222237 if ($prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
223238 # Same bytes, just an mtime touch -- flag ts-only
224239 $status = 'modified';
225240 }
226241 }
227242
228243 # Fetch + store blob only if we don't already have it
229244 if (!($have && $have->{blob_sha})) {
230245 my $cmd = 'drift-read ' . _sh_quote($p);
231246 my ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
232247 if ($c_rc != 0) {
233248 $cmd = 'cat ' . _sh_quote($p);
234249 ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
235250 }
236251 if ($c_rc != 0) {
237252 print "[$ts] fetch $p on $server->{server_name}: $c_err\n";
238253 next;
239254 }
240255 # Verify the fetched content matches the reported hash
241256 my $verify_sha = sha256_hex($c_out);
242257 if ($verify_sha ne $sha) {
243258 print "[$ts] $p: hash mismatch (list=$sha vs read=$verify_sha), using read\n";
244259 $sha = $verify_sha;
245260 }
246261 _store_blob($db, $dbh, $sha, $c_out);
247262 }
248263
249264 # Determine timestamp-only via prev SHA compare
250265 my $is_ts_only = 0;
251266 if ($prev_row && $prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
252267 $is_ts_only = 1;
253268 }
254269
255270 # Insert FILE_CHANGES row keyed to this server
256271 my $q_file = $dbh->quote($p);
257272 my $q_sha = $dbh->quote($sha);
258273 my $q_stat = $dbh->quote($status);
259274 my $mt = $remote{$p}{mtime};
260275 $db->db_readwrite($dbh, qq{
261276 INSERT INTO FILE_CHANGES
262277 (file_monitor_list_id, server_id, container_target_id,
263278 file_name, blob_sha, is_ts_only, date_time, status)
264279 VALUES
265280 ($scan->{file_monitor_list_id}, $sid, NULL,
266281 $q_file, $q_sha, $is_ts_only,
267282 FROM_UNIXTIME($mt), $q_stat)
268283 }, __FILE__, __LINE__);
269284
270285 if ($is_ts_only) { }
271286 elsif ($status eq 'added') { $total_added++; }
272287 else { $total_modified++; }
273288 }
274289
275290 # ---- Delete detection --------------------------------------
276291 foreach my $p (keys %prev) {
277292 next if $remote{$p};
278293 next if ($prev{$p}{status} // '') eq 'deleted';
279294 my $q_file = $dbh->quote($p);
280295 $db->db_readwrite($dbh, qq{
281296 INSERT INTO FILE_CHANGES
282297 (file_monitor_list_id, server_id, file_name, status, date_time)
283298 VALUES
284299 ($scan->{file_monitor_list_id}, $sid, $q_file, 'deleted', NOW())
285300 }, __FILE__, __LINE__);
286301 $total_deleted++;
287302 }
288303
289304 $db->db_readwrite($dbh, qq{
290305 UPDATE FILE_MONITOR_SETTINGS
291306 SET last_scanned = NOW()
292307 WHERE file_monitor_list_id = $scan->{file_monitor_list_id}
293308 }, __FILE__, __LINE__);
294309 }
295310
296311 # Server-level bookkeeping
297312 my $q_err = $dbh->quote($server_err || '');
298313 $db->db_readwrite($dbh, qq{
299314 UPDATE SERVERS
300315 SET last_agent_scan_at = NOW(),
301316 last_agent_error = $q_err,
302317 last_heartbeat_at = NOW()
303318 WHERE server_id = $sid
304319 }, __FILE__, __LINE__);
320
321 # Clean up decrypted-key temp file
322 unlink $tmp_key if $tmp_key && $tmp_key =~ m!^/tmp/\.ds_agent_key_!;
305323}
306324
307325$db->db_disconnect($dbh);
308326print "[$ts] agent scan complete: +$total_added added, ~$total_modified modified, -$total_deleted deleted\n"
309327 if ($total_added + $total_modified + $total_deleted) > 0;
310328exit 0;
311329
312330#---------------------------------------------------------------------
313331sub _ssh_argv {
314332 my ($user, $host, $port, $key, $opts) = @_;
315333 my @a = ('/usr/bin/ssh',
316334 '-o', 'BatchMode=yes',
317335 '-o', 'StrictHostKeyChecking=no',
318336 '-o', 'UserKnownHostsFile=/dev/null',
319337 '-o', 'LogLevel=ERROR',
320338 '-o', 'ConnectTimeout=8',
321339 '-p', $port);
322340 if ($key && -r $key) {
323341 push @a, '-i', $key;
324342 }
325343 if ($opts) {
326344 # Space-separated -o key=value pairs
327345 foreach my $o (split /\s+/, $opts) {
328346 push @a, '-o', $o if length $o;
329347 }
330348 }
331349 push @a, "$user\@$host";
332350 return @a;
333351}
334352
335353sub _ssh_run {
336354 my ($base, $cmd) = @_;
337355 my @argv = (@$base, $cmd);
338356 my ($wtr, $rdr, $err_fh);
339357 $err_fh = gensym;
340358 my $pid = eval { open3($wtr, $rdr, $err_fh, @argv) };
341359 return (-1, '', $@) if $@ || !$pid;
342360 close $wtr;
343361 binmode $rdr;
344362 my $out = do { local $/; <$rdr> // '' };
345363 my $err = do { local $/; <$err_fh> // '' };
346364 close $rdr; close $err_fh;
347365 waitpid $pid, 0;
348366 my $rc = $? >> 8;
349367 return ($rc, $out, $err);
368}
369
370sub _materialize_key {
371 my ($pem) = @_;
372 return undef unless defined $pem && length $pem;
373 my $t = POSIX::strftime('%s', localtime);
374 my $path = "/tmp/.ds_agent_key_${t}_$$_" . int(rand(1_000_000));
375 sysopen(my $fh, $path,
376 Fcntl::O_WRONLY() | Fcntl::O_CREAT() | Fcntl::O_EXCL(),
377 0600) or return undef;
378 print $fh $pem;
379 close $fh;
380 return $path;
350381}
351382
352383sub _sh_quote {
353384 my $s = shift; $s //= '';
354385 # Single-quote and escape internal single quotes
355386 $s =~ s/'/'\\''/g;
356387 return "'$s'";
357388}
358389
359390sub _build_ignore_patterns {
360391 my $list = shift;
361392 my %out;
362393 return %out unless $list;
363394 for my $ig (split /[,\n]+/, $list) {
364395 $ig =~ s/^\s+|\s+$//g;
365396 next unless length $ig;
366397 my $re = quotemeta $ig;
367398 $re =~ s/\\\*/.*/g;
368399 $re =~ s/\\\?/./g;
369400 $out{$ig} = qr/$re/;
370401 }
371402 return %out;
372403}
373404
374405sub _build_type_filter {
375406 my $list = shift;
376407 my %out;
377408 return %out unless $list && length $list;
378409 for my $ext (split /,/, $list) {
379410 $ext =~ s/^\s+|\s+$//g;
380411 next unless length $ext;
381412 $ext = ".$ext" unless $ext =~ /^\./;
382413 $out{lc $ext} = 1;
383414 }
384415 return %out;
385416}
386417
387418sub _store_blob {
388419 my ($db, $dbh, $sha, $content) = @_;
389420 my $existing = $db->db_readwrite($dbh,
390421 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
391422 __FILE__, __LINE__);
392423 if ($existing && $existing->{blob_sha}) {
393424 $db->db_readwrite($dbh,
394425 "UPDATE BLOB_STORE SET ref_count = ref_count + 1 WHERE blob_sha = " . $dbh->quote($sha),
395426 __FILE__, __LINE__);
396427 return;
397428 }
398429 my $gz = compress($content) // '';
399430 my $sth = $dbh->prepare(qq{
400431 INSERT INTO BLOB_STORE (blob_sha, content_gz, size_uncompressed, size_compressed, ref_count, first_seen_at)
401432 VALUES (?, ?, ?, ?, 1, NOW())
402433 });
403434 if ($sth) {
404435 $sth->execute($sha, $gz, length($content), length($gz));
405436 $sth->finish;
406437 }
407438}