fix(o365): stop alert floods from nine Office 365 rules

plugins/alerts/o365_action_result_rules_test.go

@@ -283,7 +283,7 @@ func TestO365ActionResultSDKHistory(t *testing.T) {
 		terms        map[string]string
 	}{
 		{"possible_succesfull_password_guessing_o365", "1m", 10, map[string]string{"action": "UserLoginFailed", "origin.user": "[email protected]", "origin.ip": "198.51.100.10"}},
-		{"credential_access_microsoft_365_potential_password_spraying_attack", "60s", 5, map[string]string{"origin.ip": "198.51.100.10"}},
+		{"credential_access_microsoft_365_potential_password_spraying_attack", "10m", 50, map[string]string{"action": "UserLoginFailed", "origin.ip": "198.51.100.10"}},
 		{"safe_links_click_patterns", "30m", 5, map[string]string{"origin.user": "[email protected]", "action": "ClickedSafeLink"}},
 		{"information_barriers_violations", "12h", 3, map[string]string{"origin.user": "[email protected]", "log.PolicyType": "InformationBarrier"}},
 	} {

plugins/alerts/o365_alert_volume_test.go

@@ -0,0 +1,312 @@
+package main
+
+// These fabricated raw records run through the checked-in O365 filter model and
+// the pinned SDK CEL and history implementation. They pin the volume thresholds
+// and de-duplication keys that keep these rules from flooding. The history
+// transport is a loopback mock, not customer storage; the EventProcessor
+// playground separately runs the real parser, history and alert plugins.
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	sdkos "github.com/threatwinds/go-sdk/os"
+	"github.com/threatwinds/go-sdk/plugins"
+	"github.com/tidwall/gjson"
+)
+
+func o365VolumeRaw(t *testing.T, fields map[string]any) string {
+	t.Helper()
+	event := map[string]any{
+		"CreationTime": "2026-09-29T10:00:00", "Id": "3c7c1f5e-9a55-4c1e-9f7b-000000000001",
+		"OrganizationId": "11111111-2222-4333-8444-555555555555", "ResultStatus": "Succeeded",
+		"UserKey": "10030000A0000001", "UserType": 0, "Version": 1,
+	}
+	for k, v := range fields {
+		event[k] = v
+	}
+	b, err := json.Marshal(event)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func o365VolumeFile(user, operation, workload string) map[string]any {
+	return map[string]any{
+		"Operation": operation, "RecordType": 6, "Workload": workload, "ClientIP": "198.51.100.22",
+		"UserId": user, "ItemType": "File", "EventSource": "SharePoint",
+		"ObjectId":          "https://contoso-my.sharepoint.com/personal/reader_example_test/Documents/plan.docx",
+		"SiteUrl":           "https://contoso-my.sharepoint.com/personal/reader_example_test/",
+		"SourceFileName":    "plan.docx",
+		"SourceRelativeUrl": "Documents",
+	}
+}
+
+func o365VolumeMailbox(user, operation string, recordType int) map[string]any {
+	return map[string]any{
+		"Operation": operation, "RecordType": recordType, "Workload": "Exchange", "ClientIP": "198.51.100.23",
+		"ClientIPAddress": "198.51.100.23", "UserId": user, "MailboxOwnerUPN": user,
+		"LogonType": 0, "OperationCount": 1,
+		"OperationProperties": []map[string]any{{"Name": "MailAccessType", "Value": "Bind"}},
+	}
+}
+
+func o365VolumeDirectory(operation string) map[string]any {
+	return map[string]any{
+		"Operation": operation, "RecordType": 8, "Workload": "AzureActiveDirectory", "ResultStatus": "Success",
+		"UserId": "[email protected]", "ObjectId": "[email protected]",
+		"ModifiedProperties": []map[string]any{{"Name": "Role.DisplayName", "NewValue": "Global Administrator", "OldValue": ""}},
+	}
+}
+
+func o365VolumeTeams() map[string]any {
+	return map[string]any{
+		"Operation": "MessagesListed", "RecordType": 25, "Workload": "MicrosoftTeams", "UserType": 5,
+		"UserId":           "backup-app",
+		"AppAccessContext": map[string]any{"ClientAppId": "0b3d7a52-8f4e-4d2b-9c61-000000000009"},
+	}
+}
+
+func o365VolumeLogin() map[string]any {
+	return map[string]any{
+		"Operation": "UserLoginFailed", "RecordType": 15, "Workload": "AzureActiveDirectory",
+		"ClientIP": "198.51.100.10", "UserId": "[email protected]",
+	}
+}
+
+func o365VolumeAudit(operation string) map[string]any {
+	return map[string]any{
+		"Operation": operation, "RecordType": 18, "Workload": "SecurityComplianceCenter", "UserType": 2,
+		"UserId": "[email protected]", "ClientIP": "198.51.100.77",
+	}
+}
+
+// Each changed rule: the fields its alerts are de-duplicated by, the raw records
+// its condition must accept, and the raw records it must now ignore.
+var o365VolumeRules = []struct {
+	file     string
+	dedup    []string
+	positive []map[string]any
+	negative []map[string]any
+}{
+	{"o365_mailbox_mass_access", []string{"adversary.user"},
+		[]map[string]any{o365VolumeMailbox("[email protected]", "MailItemsAccessed", 50)},
+		[]map[string]any{o365VolumeMailbox("[email protected]", "MailboxLogin", 2)}},
+	{"onedrive_mass_file_access", []string{"adversary.user"},
+		[]map[string]any{o365VolumeFile("[email protected]", "FileAccessed", "OneDrive"),
+			o365VolumeFile("[email protected]", "FileAccessedExtended", "OneDrive"),
+			o365VolumeFile("[email protected]", "FilePreviewed", "OneDrive")},
+		[]map[string]any{o365VolumeFile("[email protected]", "FileAccessed", "SharePoint"),
+			o365VolumeFile("[email protected]", "FileModified", "OneDrive")}},
+	{"sharepoint_mass_downloads", []string{"adversary.user"},
+		[]map[string]any{o365VolumeFile("[email protected]", "FileDownloaded", "OneDrive"),
+			o365VolumeFile("[email protected]", "FileDownloaded", "SharePoint")},
+		[]map[string]any{o365VolumeFile("[email protected]", "FileAccessed", "SharePoint")}},
+	{"teams_data_exfiltration", []string{"adversary.user", "lastEvent.log.appAccessContextClientAppId"},
+		[]map[string]any{o365VolumeTeams()},
+		[]map[string]any{{"Operation": "ChatCreated", "RecordType": 25, "Workload": "MicrosoftTeams", "UserId": "backup-app"}}},
+	{"mass_email_deletion", []string{"adversary.user"},
+		[]map[string]any{o365VolumeMailbox("[email protected]", "SoftDelete", 3),
+			o365VolumeMailbox("[email protected]", "HardDelete", 3)},
+		[]map[string]any{o365VolumeMailbox("[email protected]", "MoveToDeletedItems", 3)}},
+	{"o365-audit-log-purge", []string{"adversary.user", "lastEvent.action"},
+		[]map[string]any{o365VolumeAudit("DSIPurgeStarted"), o365VolumeAudit("AuditSearchDeleted")},
+		[]map[string]any{o365VolumeMailbox("[email protected]", "HardDelete", 3),
+			{"Operation": "Set-AdminAuditLogConfig", "RecordType": 1, "ResultStatus": "True", "Workload": "Exchange", "UserId": "[email protected]"}}},
+	{"credential_access_microsoft_365_potential_password_spraying_attack", []string{"adversary.ip"},
+		[]map[string]any{o365VolumeLogin()},
+		[]map[string]any{{"Operation": "UserLoggedIn", "RecordType": 15, "Workload": "AzureActiveDirectory", "ClientIP": "198.51.100.10", "UserId": "[email protected]"}}},
+	{"o365-admin-role-assignment", []string{"adversary.user", "lastEvent.log.ObjectId"},
+		[]map[string]any{o365VolumeDirectory("Add member to role.")},
+		[]map[string]any{o365VolumeDirectory("Add member to group."), o365VolumeDirectory("Add delegated permission grant."),
+			o365VolumeDirectory("Update user.")}},
+}
+
+func TestO365AlertVolumePredicatesAndKeys(t *testing.T) {
+	cache := plugins.NewCELCache("o365-alert-volume")
+	for _, tc := range o365VolumeRules {
+		t.Run(tc.file, func(t *testing.T) {
+			r := o365OutcomeRule(t, tc.file)
+			if len(r.GroupBy) != 0 || !reflect.DeepEqual(r.DeduplicateBy, tc.dedup) {
+				t.Fatalf("grouping got groupBy %v deduplicateBy %v, want deduplicateBy %v", r.GroupBy, r.DeduplicateBy, tc.dedup)
+			}
+			for i, fields := range tc.positive {
+				event := o365ActionResultNormalize(t, o365VolumeRaw(t, fields))
+				if got, err := cache.Eval(r.Where, event); err != nil || !got {
+					t.Fatalf("positive %d: got %v %v for %s", i, got, err, event)
+				}
+			}
+			for i, fields := range tc.negative {
+				event := o365ActionResultNormalize(t, o365VolumeRaw(t, fields))
+				if got, err := cache.Eval(r.Where, event); err != nil || got {
+					t.Fatalf("negative %d: got %v %v for %s", i, got, err, event)
+				}
+			}
+		})
+	}
+	if _, err := os.Stat("../../rules/office365/o365-admin-role-granted.yml"); !os.IsNotExist(err) {
+		t.Fatal("the Update user. proxy rule must stay removed; role assignments are O365 Admin Role Assignment")
+	}
+}
+
+func TestO365AlertVolumeSDKHistory(t *testing.T) {
+	// Isolate the SDK's process-global OpenSearch connection and field mapper.
+	if os.Getenv("UTM_O365_VOLUME_HISTORY_CHILD") != "1" {
+		c := exec.Command(os.Args[0], "-test.run=^TestO365AlertVolumeSDKHistory$")
+		c.Env = append(os.Environ(), "UTM_O365_VOLUME_HISTORY_CHILD=1")
+		if b, err := c.CombinedOutput(); err != nil {
+			t.Fatalf("isolated history: %v\n%s", err, b)
+		}
+		return
+	}
+	var history []string
+	var expectedTerms map[string]string
+	var window time.Duration
+	mapping := map[string]any{"properties": map[string]any{
+		"@timestamp": map[string]any{"type": "date"}, "action": map[string]any{"type": "keyword"},
+		"origin": map[string]any{"properties": map[string]any{"user": map[string]any{"type": "keyword"}, "ip": map[string]any{"type": "ip"}}},
+		"log":    map[string]any{"properties": map[string]any{"Workload": map[string]any{"type": "keyword"}}},
+	}}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "/_mapping") {
+			_ = json.NewEncoder(w).Encode(map[string]any{"v11-log-o365-test": map[string]any{"mappings": mapping}})
+			return
+		}
+		if r.URL.Path != "/v11-log-o365-*/_search" {
+			t.Errorf("unexpected request %s", r.URL.Path)
+			http.Error(w, "bad request", 400)
+			return
+		}
+		b, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		q := string(b)
+		terms := map[string]string{}
+		cutoff := time.Time{}
+		if gjson.Get(q, "query.bool.must_not").Exists() {
+			t.Error("unexpected negative history clauses")
+		}
+		for _, clause := range append(gjson.Get(q, "query.bool.filter").Array(), gjson.Get(q, "query.bool.must").Array()...) {
+			if term := clause.Get("term"); term.Exists() {
+				for field, value := range term.Map() {
+					terms[strings.TrimSuffix(field, ".keyword")] = value.Get("value").String()
+				}
+			} else if span := clause.Get("range"); span.Exists() {
+				if cutoff, err = time.Parse(time.RFC3339Nano, span.Get("@timestamp.gte").String()); err != nil {
+					t.Error(err)
+				}
+			} else {
+				t.Errorf("unsupported history clause %s", clause.Raw)
+			}
+		}
+		if !reflect.DeepEqual(terms, expectedTerms) {
+			t.Errorf("history scope got %v want %v", terms, expectedTerms)
+		}
+		if delta := time.Since(cutoff) - window; delta < -2*time.Second || delta > 2*time.Second {
+			t.Errorf("wrong cutoff %v", delta)
+		}
+		hits := []map[string]any{}
+		for _, doc := range history {
+			match := true
+			for field, value := range terms {
+				if gjson.Get(doc, field).String() != value {
+					match = false
+				}
+			}
+			stamp, err := time.Parse(time.RFC3339Nano, gjson.Get(doc, "@timestamp").String())
+			if err != nil || stamp.Before(cutoff) {
+				match = false
+			}
+			if match {
+				hits = append(hits, map[string]any{"_id": fmt.Sprint(len(hits)), "_index": "v11-log-o365-test", "_source": map[string]any{}})
+			}
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"took": 1, "hits": map[string]any{"total": map[string]any{"value": len(hits), "relation": "eq"}, "hits": hits}})
+	}))
+	defer server.Close()
+	if err := sdkos.Connect([]string{server.URL}, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	// Every history search, parent first and then its "or" branches in order:
+	// the raw record whose event is counted, the window, the threshold and the scope.
+	type search struct {
+		fields map[string]any
+		within string
+		count  uint64
+		terms  map[string]string
+	}
+	user := "[email protected]"
+	for _, tc := range []struct {
+		file     string
+		searches []search
+	}{
+		{"o365_mailbox_mass_access", []search{
+			{o365VolumeMailbox(user, "MailItemsAccessed", 50), "1h", 2000, map[string]string{"origin.user": user, "action": "MailItemsAccessed"}}}},
+		{"onedrive_mass_file_access", []search{
+			{o365VolumeFile(user, "FileAccessed", "OneDrive"), "1h", 1000, map[string]string{"origin.user": user, "action": "FileAccessed", "log.Workload": "OneDrive"}},
+			{o365VolumeFile(user, "FileAccessedExtended", "OneDrive"), "1h", 1000, map[string]string{"origin.user": user, "action": "FileAccessedExtended", "log.Workload": "OneDrive"}},
+			{o365VolumeFile(user, "FilePreviewed", "OneDrive"), "1h", 1000, map[string]string{"origin.user": user, "action": "FilePreviewed", "log.Workload": "OneDrive"}}}},
+		{"sharepoint_mass_downloads", []search{
+			{o365VolumeFile(user, "FileDownloaded", "SharePoint"), "1h", 500, map[string]string{"origin.user": user, "action": "FileDownloaded"}}}},
+		{"teams_data_exfiltration", []search{
+			{o365VolumeTeams(), "1h", 100, map[string]string{"origin.user": "backup-app", "log.Workload": "MicrosoftTeams"}}}},
+		{"mass_email_deletion", []search{
+			{o365VolumeMailbox("[email protected]", "HardDelete", 3), "1h", 200, map[string]string{"origin.user": "[email protected]", "action": "HardDelete"}},
+			{o365VolumeMailbox("[email protected]", "SoftDelete", 3), "1h", 5000, map[string]string{"origin.user": "[email protected]", "action": "SoftDelete"}}}},
+		{"credential_access_microsoft_365_potential_password_spraying_attack", []search{
+			{o365VolumeLogin(), "10m", 50, map[string]string{"origin.ip": "198.51.100.10", "action": "UserLoginFailed"}}}},
+	} {
+		t.Run(tc.file, func(t *testing.T) {
+			r := o365OutcomeRule(t, tc.file)
+			if len(r.Correlation) != 1 || len(r.Correlation[0].Or) != len(tc.searches)-1 {
+				t.Fatalf("unexpected history shape: %d searches", len(r.Correlation))
+			}
+			searches := append([]*plugins.SearchRequest{r.Correlation[0]}, r.Correlation[0].Or...)
+			for i, want := range tc.searches {
+				s := searches[i]
+				if s.Count != want.count || s.Within != want.within || len(s.Or) != 0 && i > 0 {
+					t.Fatalf("search %d: count %d within %s, want %d %s", i, s.Count, s.Within, want.count, want.within)
+				}
+				event := o365ActionResultNormalize(t, o365VolumeRaw(t, want.fields))
+				expectedTerms = want.terms
+				window, _ = time.ParseDuration(want.within)
+				// The branch is executed alone; the parent's own "or" list is checked above.
+				alone := &plugins.SearchRequest{IndexPattern: s.IndexPattern, With: s.With, Within: s.Within, Count: s.Count}
+				prior := o365OutcomeSet(t, event, "@timestamp", time.Now().Add(-window/2).UTC().Format(time.RFC3339Nano))
+				history = nil
+				for n := uint64(0); n < want.count-1; n++ {
+					history = append(history, prior)
+				}
+				if yes, _, err := alone.Execute(&event); err != nil || yes {
+					t.Fatalf("search %d below threshold: %v %v", i, yes, err)
+				}
+				history = append(history, prior)
+				if yes, _, err := alone.Execute(&event); err != nil || !yes {
+					t.Fatalf("search %d at threshold: %v %v", i, yes, err)
+				}
+				history[len(history)-1] = o365OutcomeSet(t, prior, "@timestamp", time.Now().Add(-window-time.Minute).UTC().Format(time.RFC3339Nano))
+				if yes, _, err := alone.Execute(&event); err != nil || yes {
+					t.Fatalf("search %d counted expired history: %v %v", i, yes, err)
+				}
+				for field := range want.terms {
+					history[len(history)-1] = o365OutcomeSet(t, prior, field, "different-value")
+					if yes, _, err := alone.Execute(&event); err != nil || yes {
+						t.Fatalf("search %d counted a different %s: %v %v", i, field, yes, err)
+					}
+				}
+			}
+		})
+	}
+}

rules/office365/credential_access_microsoft_365_potential_password_spraying_attack.yml

@@ -1,4 +1,4 @@
-# Rule version v1.0.6
+# Rule version v1.1.0
 
 dataTypes:
   - "o365"
@@ -10,11 +10,11 @@ impact:
 category: "Credential Access"
 technique: "T1110 - Brute Force"
 adversary: origin
-references: 
+references:
   - "https://attack.mitre.org/techniques/T1110/"
   - "https://attack.mitre.org/tactics/TA0006/"
 description: "Credential Access consists of techniques for stealing credentials like account names and passwords. Techniques used to get credentials include keylogging or credential dumping. Using legitimate credentials can give adversaries access to systems, make them harder to detect, and provide the opportunity to create more accounts to help achieve their goals.<br>
-  Identifies a high number (25) of failed Microsoft 365 user authentication attempts from a single IP address within 30 minutes, which could be indicative of a password spraying attack. An adversary may attempt a password spraying attack to obtain unauthorized access to user accounts."
+  Identifies a high number (50) of failed Microsoft 365 sign-ins (UserLoginFailed) from a single IP address within 10 minutes, which could be indicative of a password spraying attack. An adversary may attempt a password spraying attack to obtain unauthorized access to user accounts. Company internet gateways carry many ordinary failures, such as multi-factor prompts and expired sessions, so the threshold is set above that everyday volume. One alert is raised per IP address; repeats for the same address are suppressed for seven days."
 where: |
   oneOf("log.Workload", ["Exchange", "AzureActiveDirectory"]) && oneOf("action", ["UserLoginFailed", "PasswordLogonInitialAuthUsingPassword"]) && oneOf("actionResult", ["failed", "denied"]) && exists("origin.ip")
 afterEvents:
@@ -23,7 +23,10 @@ afterEvents:
       - field: origin.ip
         operator: filter_term
         value: '{{.origin.ip}}'
-    within: 60s
-    count: 5
-groupBy:
+      - field: action
+        operator: filter_term
+        value: 'UserLoginFailed'
+    within: 10m
+    count: 50
+deduplicateBy:
   - adversary.ip

rules/office365/mass_email_deletion.yml

@@ -1,4 +1,4 @@
-# Rule version v1.0.0
+# Rule version v1.1.0
 
 dataTypes:
   - o365
@@ -12,10 +12,11 @@ technique: "T1114 - Email Collection"
 adversary: origin
 references:
   - https://learn.microsoft.com/en-us/purview/audit-mailboxes
+  - https://learn.microsoft.com/en-us/purview/audit-log-activities
   - https://attack.mitre.org/techniques/T1114/
 description: |
-  Detects when a user performs mass deletion of emails which could indicate data destruction, covering tracks, or malicious insider activity. Monitors for multiple HardDelete or SoftDelete operations within a short time window.
-  
+  Detects when a user performs mass deletion of emails which could indicate data destruction, covering tracks, or malicious insider activity. Triggers when one user writes 200 or more HardDelete records (messages purged from the Recoverable Items folder) or 5,000 or more SoftDelete records (messages permanently deleted or removed from Deleted Items) within one hour. Emptying a large Deleted Items folder in Outlook produces many SoftDelete records, so the SoftDelete threshold is set far above everyday cleanup. One alert is raised per user; repeats for the same user are suppressed for seven days.
+
   Next Steps:
   1. Verify the legitimacy of the user performing the deletions
   2. Check if this aligns with any scheduled maintenance or cleanup activities
@@ -35,8 +36,8 @@ afterEvents:
       - field: action
         operator: filter_term
         value: 'HardDelete'
-    within: 15m
-    count: 100
+    within: 1h
+    count: 200
     or:
       - indexPattern: v11-log-o365-*
         with:
@@ -46,7 +47,7 @@ afterEvents:
           - field: action
             operator: filter_term
             value: 'SoftDelete'
-        within: 15m
-        count: 100
-groupBy:
+        within: 1h
+        count: 5000
+deduplicateBy:
   - adversary.user