Diff -- /var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

O Operator
Diff

/var/www/vhosts/3dshawn.com/site1/_agent_scan.pl

added on local at 2026-07-11 23:03:27

Added
+407
lines
Removed
-0
lines
Context
0
unchanged
Blobs
from
to e833d76d304c
Restore this content →
Unified diff
Naive LCS-based line diff. Additions in emerald, deletions in rose. Both sides decompressed live from BLOB_STORE.
1#!/usr/bin/perl
2#======================================================================
3# DriftSense -- SSH agent scanner (Stealth Flavor 2)
4#
5# Cron entry point (NOT a CGI). For each active SERVERS row with
6# kind='ssh_agent', SSHes into the monitored host using the configured
7# restricted key and walks the paths in that server's FILE_MONITOR_SETTINGS.
8#
9# Monitored host runs zero DriftSense code -- only a single command-
10# restricted authorized_keys entry. The wrapper on the remote side
11# only allows 3 commands:
12#
13# drift-find <path> -> path\tsize\tmtime\n per file
14# drift-hash <files> -> sha256sum output
15# drift-read <file> -> raw bytes
16#
17# We drive it via SSH_ORIGINAL_COMMAND. If the wrapper isn't installed,
18# we fall back to plain `find`, `sha256sum`, `cat` on the assumption
19# that the SSH user has a normal shell (test-mode).
20#
21# Config: /etc/drift_sense/drift_sense.conf (via MODS::Config).
22# Log: /var/log/drift_sense/agent_scan.log (via cron redirect).
23#======================================================================
24use strict;
25use warnings;
26use lib '/var/www/vhosts/3dshawn.com/site1';
27use POSIX ();
28use Digest::SHA qw(sha256_hex);
29use Compress::Zlib qw(compress);
30use IPC::Open3;
31use Symbol 'gensym';
32use MODS::Config;
33use MODS::DBConnect;
34
35$| = 1;
36my $ts = POSIX::strftime('%Y-%m-%d %H:%M:%S', localtime);
37
38my $cfg = MODS::Config->new;
39my $db = MODS::DBConnect->new;
40my $dbh = $db->db_connect or die "[$ts] cannot connect to DriftSense storage DB\n";
41
42my $default_max_size = int($cfg->settings('max_file_size_bytes') || 10_485_760);
43my $DEFAULT_TIMEOUT = 30;
44
45# ---- Fetch ssh_agent servers ----------------------------------------
46my @servers = $db->db_readwrite_multiple($dbh, q~
47 SELECT server_id, server_name, ssh_host, ssh_user, ssh_port,
48 ssh_key_path, ssh_options
49 FROM SERVERS
50 WHERE kind = 'ssh_agent'
51 AND status = 'active'
52~, __FILE__, __LINE__);
53
54unless (@servers) {
55 print "[$ts] no active ssh_agent servers, exiting\n";
56 $db->db_disconnect($dbh);
57 exit 0;
58}
59
60my $total_added = 0;
61my $total_modified = 0;
62my $total_deleted = 0;
63
64foreach my $server (@servers) {
65 my $sid = $server->{server_id};
66 my $host = $server->{ssh_host} || next;
67 my $user = $server->{ssh_user} || 'root';
68 my $port = $server->{ssh_port} || 22;
69 my $key = $server->{ssh_key_path} || '';
70 my $opts = $server->{ssh_options} || '';
71
72 my @ssh_base = _ssh_argv($user, $host, $port, $key, $opts);
73
74 # Fetch this server's active file monitors
75 my @scans = $db->db_readwrite_multiple($dbh, qq~
76 SELECT file_monitor_list_id, scan_path, ignore_list, file_type_filter,
77 max_file_size_bytes, scan_name
78 FROM FILE_MONITOR_SETTINGS
79 WHERE status = 1
80 AND server_id = $sid
81 ~, __FILE__, __LINE__);
82
83 unless (@scans) {
84 print "[$ts] server '$server->{server_name}' has no active monitors\n";
85 next;
86 }
87
88 my $server_err;
89
90 foreach my $scan (@scans) {
91 my $path = $scan->{scan_path} or next;
92 my $max_size = $scan->{max_file_size_bytes} || $default_max_size;
93
94 my %ignore_pats = _build_ignore_patterns($scan->{ignore_list});
95 my %type_filter = _build_type_filter($scan->{file_type_filter});
96
97 # ---- Remote listing -----------------------------------------
98 my $remote_cmd = qq{drift-find $path};
99 my ($rc, $out, $err) = _ssh_run(\@ssh_base, $remote_cmd);
100 if ($rc != 0) {
101 # Fallback: no wrapper -> plain shell
102 my $shell = qq{find } . _sh_quote($path) . qq{ -type f -not -path '*/.*' -printf '%p\\t%s\\t%T@\\n' 2>/dev/null};
103 ($rc, $out, $err) = _ssh_run(\@ssh_base, $shell);
104 }
105 if ($rc != 0) {
106 $server_err = "list failed: $err";
107 print "[$ts] server '$server->{server_name}' scan '$scan->{scan_name}': $server_err\n";
108 next;
109 }
110
111 # Parse listing: path\tsize\tmtime_epoch
112 my %remote;
113 foreach my $line (split /\n/, $out) {
114 chomp $line;
115 next unless length $line;
116 my ($p, $sz, $mt) = split /\t/, $line, 3;
117 next unless defined $p && defined $sz && defined $mt;
118 $sz = int($sz);
119 $mt = int($mt); # printf gives float mtime, truncate
120
121 # Ignore-glob filter (local side)
122 my $skip = 0;
123 for my $pat (keys %ignore_pats) {
124 my $re = $ignore_pats{$pat};
125 if ($p =~ $re) { $skip = 1; last; }
126 }
127 next if $skip;
128
129 # File-type filter (local side)
130 if (%type_filter) {
131 my $ext_ok = 0;
132 for my $e (keys %type_filter) {
133 if (lc(substr($p, -length($e))) eq $e) { $ext_ok = 1; last; }
134 }
135 next unless $ext_ok;
136 }
137
138 next if $sz > $max_size;
139 $remote{$p} = { size => $sz, mtime => $mt };
140 }
141
142 # ---- Previous state ----------------------------------------
143 my $prev_sth = $dbh->prepare(qq{
144 SELECT fc.file_name, fc.blob_sha, fc.status,
145 UNIX_TIMESTAMP(fc.date_time) AS mt_epoch
146 FROM FILE_CHANGES fc
147 WHERE fc.file_monitor_list_id = $scan->{file_monitor_list_id}
148 AND fc.server_id = $sid
149 AND fc.file_changes_id IN (
150 SELECT MAX(inner_fc.file_changes_id)
151 FROM FILE_CHANGES inner_fc
152 WHERE inner_fc.file_monitor_list_id = $scan->{file_monitor_list_id}
153 AND inner_fc.server_id = $sid
154 GROUP BY inner_fc.file_name
155 )
156 });
157 my %prev;
158 if ($prev_sth && $prev_sth->execute) {
159 while (my $r = $prev_sth->fetchrow_hashref) {
160 $prev{$r->{file_name}} = $r;
161 }
162 $prev_sth->finish;
163 }
164
165 # ---- Decide who needs a fresh hash --------------------------
166 my @needs_hash;
167 foreach my $p (keys %remote) {
168 my $prev_row = $prev{$p};
169 if (!$prev_row || ($prev_row->{status} // '') eq 'deleted') {
170 push @needs_hash, $p;
171 next;
172 }
173 if (($prev_row->{mt_epoch} // 0) != $remote{$p}{mtime}) {
174 push @needs_hash, $p;
175 }
176 }
177
178 # ---- Batch hash over SSH -----------------------------------
179 # `drift-hash <files>` returns sha256sum output: "<sha> <path>"
180 my %fresh_sha;
181 if (@needs_hash) {
182 my $q_list = join(' ', map { _sh_quote($_) } @needs_hash);
183 my $cmd = "drift-hash $q_list";
184 my ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
185 if ($h_rc != 0) {
186 # Fallback: plain sha256sum
187 $cmd = "sha256sum $q_list 2>/dev/null";
188 ($h_rc, $h_out, $h_err) = _ssh_run(\@ssh_base, $cmd);
189 }
190 if ($h_rc == 0) {
191 foreach my $line (split /\n/, $h_out) {
192 chomp $line;
193 if ($line =~ /^([0-9a-f]{64})\s+(.+)$/) {
194 $fresh_sha{$2} = $1;
195 }
196 }
197 }
198 }
199
200 # ---- Process each remote file ------------------------------
201 foreach my $p (keys %remote) {
202 my $sha = $fresh_sha{$p};
203 my $prev_row = $prev{$p};
204
205 # If we didn't rehash this file, previous SHA still valid
206 if (!defined $sha && $prev_row && ($prev_row->{status} // '') ne 'deleted') {
207 # Nothing changed since last tick -- move on
208 next;
209 }
210
211 # Content-addressable dedup: if we already have this SHA in
212 # BLOB_STORE, don't re-fetch content over SSH.
213 my $have = $db->db_readwrite($dbh,
214 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
215 __FILE__, __LINE__);
216
217 my $status;
218 if (!$prev_row) {
219 $status = 'added';
220 } else {
221 $status = 'modified';
222 if ($prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
223 # Same bytes, just an mtime touch -- flag ts-only
224 $status = 'modified';
225 }
226 }
227
228 # Fetch + store blob only if we don't already have it
229 if (!($have && $have->{blob_sha})) {
230 my $cmd = 'drift-read ' . _sh_quote($p);
231 my ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
232 if ($c_rc != 0) {
233 $cmd = 'cat ' . _sh_quote($p);
234 ($c_rc, $c_out, $c_err) = _ssh_run(\@ssh_base, $cmd);
235 }
236 if ($c_rc != 0) {
237 print "[$ts] fetch $p on $server->{server_name}: $c_err\n";
238 next;
239 }
240 # Verify the fetched content matches the reported hash
241 my $verify_sha = sha256_hex($c_out);
242 if ($verify_sha ne $sha) {
243 print "[$ts] $p: hash mismatch (list=$sha vs read=$verify_sha), using read\n";
244 $sha = $verify_sha;
245 }
246 _store_blob($db, $dbh, $sha, $c_out);
247 }
248
249 # Determine timestamp-only via prev SHA compare
250 my $is_ts_only = 0;
251 if ($prev_row && $prev_row->{blob_sha} && $prev_row->{blob_sha} eq $sha) {
252 $is_ts_only = 1;
253 }
254
255 # Insert FILE_CHANGES row keyed to this server
256 my $q_file = $dbh->quote($p);
257 my $q_sha = $dbh->quote($sha);
258 my $q_stat = $dbh->quote($status);
259 my $mt = $remote{$p}{mtime};
260 $db->db_readwrite($dbh, qq{
261 INSERT INTO FILE_CHANGES
262 (file_monitor_list_id, server_id, container_target_id,
263 file_name, blob_sha, is_ts_only, date_time, status)
264 VALUES
265 ($scan->{file_monitor_list_id}, $sid, NULL,
266 $q_file, $q_sha, $is_ts_only,
267 FROM_UNIXTIME($mt), $q_stat)
268 }, __FILE__, __LINE__);
269
270 if ($is_ts_only) { }
271 elsif ($status eq 'added') { $total_added++; }
272 else { $total_modified++; }
273 }
274
275 # ---- Delete detection --------------------------------------
276 foreach my $p (keys %prev) {
277 next if $remote{$p};
278 next if ($prev{$p}{status} // '') eq 'deleted';
279 my $q_file = $dbh->quote($p);
280 $db->db_readwrite($dbh, qq{
281 INSERT INTO FILE_CHANGES
282 (file_monitor_list_id, server_id, file_name, status, date_time)
283 VALUES
284 ($scan->{file_monitor_list_id}, $sid, $q_file, 'deleted', NOW())
285 }, __FILE__, __LINE__);
286 $total_deleted++;
287 }
288
289 $db->db_readwrite($dbh, qq{
290 UPDATE FILE_MONITOR_SETTINGS
291 SET last_scanned = NOW()
292 WHERE file_monitor_list_id = $scan->{file_monitor_list_id}
293 }, __FILE__, __LINE__);
294 }
295
296 # Server-level bookkeeping
297 my $q_err = $dbh->quote($server_err || '');
298 $db->db_readwrite($dbh, qq{
299 UPDATE SERVERS
300 SET last_agent_scan_at = NOW(),
301 last_agent_error = $q_err,
302 last_heartbeat_at = NOW()
303 WHERE server_id = $sid
304 }, __FILE__, __LINE__);
305}
306
307$db->db_disconnect($dbh);
308print "[$ts] agent scan complete: +$total_added added, ~$total_modified modified, -$total_deleted deleted\n"
309 if ($total_added + $total_modified + $total_deleted) > 0;
310exit 0;
311
312#---------------------------------------------------------------------
313sub _ssh_argv {
314 my ($user, $host, $port, $key, $opts) = @_;
315 my @a = ('/usr/bin/ssh',
316 '-o', 'BatchMode=yes',
317 '-o', 'StrictHostKeyChecking=no',
318 '-o', 'UserKnownHostsFile=/dev/null',
319 '-o', 'LogLevel=ERROR',
320 '-o', 'ConnectTimeout=8',
321 '-p', $port);
322 if ($key && -r $key) {
323 push @a, '-i', $key;
324 }
325 if ($opts) {
326 # Space-separated -o key=value pairs
327 foreach my $o (split /\s+/, $opts) {
328 push @a, '-o', $o if length $o;
329 }
330 }
331 push @a, "$user\@$host";
332 return @a;
333}
334
335sub _ssh_run {
336 my ($base, $cmd) = @_;
337 my @argv = (@$base, $cmd);
338 my ($wtr, $rdr, $err_fh);
339 $err_fh = gensym;
340 my $pid = eval { open3($wtr, $rdr, $err_fh, @argv) };
341 return (-1, '', $@) if $@ || !$pid;
342 close $wtr;
343 binmode $rdr;
344 my $out = do { local $/; <$rdr> // '' };
345 my $err = do { local $/; <$err_fh> // '' };
346 close $rdr; close $err_fh;
347 waitpid $pid, 0;
348 my $rc = $? >> 8;
349 return ($rc, $out, $err);
350}
351
352sub _sh_quote {
353 my $s = shift; $s //= '';
354 # Single-quote and escape internal single quotes
355 $s =~ s/'/'\\''/g;
356 return "'$s'";
357}
358
359sub _build_ignore_patterns {
360 my $list = shift;
361 my %out;
362 return %out unless $list;
363 for my $ig (split /[,\n]+/, $list) {
364 $ig =~ s/^\s+|\s+$//g;
365 next unless length $ig;
366 my $re = quotemeta $ig;
367 $re =~ s/\\\*/.*/g;
368 $re =~ s/\\\?/./g;
369 $out{$ig} = qr/$re/;
370 }
371 return %out;
372}
373
374sub _build_type_filter {
375 my $list = shift;
376 my %out;
377 return %out unless $list && length $list;
378 for my $ext (split /,/, $list) {
379 $ext =~ s/^\s+|\s+$//g;
380 next unless length $ext;
381 $ext = ".$ext" unless $ext =~ /^\./;
382 $out{lc $ext} = 1;
383 }
384 return %out;
385}
386
387sub _store_blob {
388 my ($db, $dbh, $sha, $content) = @_;
389 my $existing = $db->db_readwrite($dbh,
390 "SELECT blob_sha FROM BLOB_STORE WHERE blob_sha = " . $dbh->quote($sha) . " LIMIT 1",
391 __FILE__, __LINE__);
392 if ($existing && $existing->{blob_sha}) {
393 $db->db_readwrite($dbh,
394 "UPDATE BLOB_STORE SET ref_count = ref_count + 1 WHERE blob_sha = " . $dbh->quote($sha),
395 __FILE__, __LINE__);
396 return;
397 }
398 my $gz = compress($content) // '';
399 my $sth = $dbh->prepare(qq{
400 INSERT INTO BLOB_STORE (blob_sha, content_gz, size_uncompressed, size_compressed, ref_count, first_seen_at)
401 VALUES (?, ?, ?, ?, 1, NOW())
402 });
403 if ($sth) {
404 $sth->execute($sha, $gz, length($content), length($gz));
405 $sth->finish;
406 }
407}