Оперативная память и хранилища

<?php

declare(strict_types=1);

/**
 * Memory awareness in PHP applications.
 * Understanding memory limits prevents OOM crashes.
 */
final class MemoryMonitor
{
    /**
     * Check current memory usage and proximity to limit.
     *
     * @return array{used_mb: float, limit_mb: float, usage_percent: float}
     */
    public static function getUsage(): array
    {
        $used = memory_get_usage(true);
        $peak = memory_get_peak_usage(true);
        $limit = self::parseMemoryLimit(ini_get('memory_limit') ?: '128M');

        return [
            'used_mb' => round($used / 1024 / 1024, 2),
            'peak_mb' => round($peak / 1024 / 1024, 2),
            'limit_mb' => round($limit / 1024 / 1024, 2),
            'usage_percent' => round(($used / $limit) * 100, 1),
        ];
    }

    /**
     * Process large datasets without loading everything into memory.
     * This is crucial for PHP where memory_limit is typically 128-256MB.
     *
     * @return \Generator<int, array<string, mixed>>
     */
    public static function streamLargeDataset(\PDO $pdo, string $query): \Generator
    {
        // Use unbuffered query to avoid loading all rows into memory
        $pdo->setAttribute(\PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);

        $stmt = $pdo->query($query);

        while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
            yield $row;
        }

        $pdo->setAttribute(\PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, true);
    }

    private static function parseMemoryLimit(string $limit): int
    {
        $value = (int) $limit;
        $unit = strtolower(substr($limit, -1));

        return match ($unit) {
            'g' => $value * 1024 * 1024 * 1024,
            'm' => $value * 1024 * 1024,
            'k' => $value * 1024,
            default => $value,
        };
    }
}

package memory

import (
	"context"
	"database/sql"
	"fmt"
	"iter"
	"runtime"
)

// MemoryUsage holds current memory statistics.
type MemoryUsage struct {
	UsedMB       float64
	TotalAllocMB float64
	SysMB        float64
	NumGC        uint32
}

// GetUsage returns current memory usage of the Go process.
// Go has no hard memory_limit like PHP — it uses OS virtual memory.
func GetUsage() MemoryUsage {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)

	return MemoryUsage{
		UsedMB:       float64(m.Alloc) / 1024 / 1024,
		TotalAllocMB: float64(m.TotalAlloc) / 1024 / 1024,
		SysMB:        float64(m.Sys) / 1024 / 1024,
		NumGC:        m.NumGC,
	}
}

// StreamLargeDataset processes rows one at a time using Go 1.23+ iterators.
// Unlike PHP generators, Go uses iter.Seq for lazy evaluation.
func StreamLargeDataset(ctx context.Context, db *sql.DB, query string) iter.Seq2[map[string]any, error] {
	return func(yield func(map[string]any, error) bool) {
		rows, err := db.QueryContext(ctx, query)
		if err != nil {
			yield(nil, fmt.Errorf("query: %w", err))
			return
		}
		defer rows.Close()

		cols, err := rows.Columns()
		if err != nil {
			yield(nil, fmt.Errorf("columns: %w", err))
			return
		}

		for rows.Next() {
			values := make([]any, len(cols))
			ptrs := make([]any, len(cols))
			for i := range values {
				ptrs[i] = &values[i]
			}

			if err := rows.Scan(ptrs...); err != nil {
				yield(nil, fmt.Errorf("scan: %w", err))
				return
			}

			row := make(map[string]any, len(cols))
			for i, col := range cols {
				row[col] = values[i]
			}

			if !yield(row, nil) {
				return
			}
		}
	}
}

using System.Data;
using System.Data.Common;
using System.Diagnostics;
using System.Runtime.CompilerServices;

namespace Foundations.Memory;

public readonly record struct MemoryUsage(
    double UsedMb,
    double PeakMb,
    double LimitMb,
    double UsagePercent);

/// <summary>
/// Memory awareness in .NET applications.
/// Understanding memory limits prevents OOM crashes.
/// </summary>
public static class MemoryMonitor
{
    /// <summary>
    /// Reports current managed heap usage against the GC hard limit.
    /// Unlike PHP there is no memory_limit ini setting - the ceiling comes
    /// from the GC configuration or the container cgroup limit.
    /// </summary>
    public static MemoryUsage GetUsage()
    {
        var info = GC.GetGCMemoryInfo();
        var used = GC.GetTotalMemory(forceFullCollection: false);
        var peak = Process.GetCurrentProcess().PeakWorkingSet64;
        var limit = info.TotalAvailableMemoryBytes;

        return new MemoryUsage(
            UsedMb: Math.Round(used / 1024d / 1024d, 2),
            PeakMb: Math.Round(peak / 1024d / 1024d, 2),
            LimitMb: Math.Round(limit / 1024d / 1024d, 2),
            UsagePercent: Math.Round((double)used / limit * 100, 1));
    }

    /// <summary>
    /// Process large datasets without loading everything into memory.
    /// The data reader streams rows straight from the wire.
    /// </summary>
    public static async IAsyncEnumerable<Dictionary<string, object?>> StreamLargeDatasetAsync(
        DbConnection connection,
        string query,
        [EnumeratorCancellation] CancellationToken ct = default)
    {
        await using var command = connection.CreateCommand();
        command.CommandText = query;

        // SequentialAccess avoids buffering the whole row in memory
        await using var reader = await command.ExecuteReaderAsync(
            CommandBehavior.SequentialAccess, ct);

        while (await reader.ReadAsync(ct))
        {
            var row = new Dictionary<string, object?>(reader.FieldCount);
            for (var i = 0; i < reader.FieldCount; i++)
            {
                row[reader.GetName(i)] = await reader.IsDBNullAsync(i, ct)
                    ? null
                    : reader.GetValue(i);
            }

            yield return row;
        }
    }
}

from __future__ import annotations

import os
import resource
from collections.abc import Iterator
from typing import Any

import psutil
from psycopg import Connection

def get_usage() -> dict[str, float]:
    """Report process memory usage against the configured limit.

    Python has no memory_limit ini setting: the ceiling comes from the OS
    (RLIMIT_AS) or from the container cgroup, so read it from there.
    """
    process = psutil.Process(os.getpid())
    used = process.memory_info().rss
    peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
    soft_limit, _ = resource.getrlimit(resource.RLIMIT_AS)
    limit = soft_limit if soft_limit != resource.RLIM_INFINITY else psutil.virtual_memory().total

    return {
        "used_mb": round(used / 1024 / 1024, 2),
        "peak_mb": round(peak / 1024 / 1024, 2),
        "limit_mb": round(limit / 1024 / 1024, 2),
        "usage_percent": round(used / limit * 100, 1),
    }

def stream_large_dataset(connection: Connection, query: str) -> Iterator[dict[str, Any]]:
    """Yield rows one at a time instead of materializing the whole result set.

    A server-side (named) cursor keeps the result on the database side and
    fetches it in chunks, so memory stays flat regardless of row count.
    """
    with connection.cursor(name="stream_large_dataset") as cursor:
        cursor.itersize = 1000
        cursor.execute(query)
        columns = [description.name for description in cursor.description]

        for row in cursor:
            yield dict(zip(columns, row, strict=True))

<?php

declare(strict_types=1);

/**
 * Cache-friendly data access patterns matter even in PHP.
 * Sequential access is much faster than random due to CPU cache behavior.
 */
final class CacheAwareProcessing
{
    /**
     * Sequential access: cache-friendly.
     * CPU prefetcher predicts the pattern and preloads data.
     */
    public static function processSequential(array $data): float
    {
        $sum = 0.0;
        $count = count($data);

        for ($i = 0; $i < $count; $i++) {
            $sum += $data[$i];
        }

        return $sum;
    }

    /**
     * Random access: cache-unfriendly.
     * Each access likely causes a cache miss.
     */
    public static function processRandom(array $data): float
    {
        $sum = 0.0;
        $count = count($data);
        $indices = range(0, $count - 1);
        shuffle($indices);

        foreach ($indices as $i) {
            $sum += $data[$i];
        }

        return $sum;
    }

    /**
     * Structure of Arrays (SoA) vs Array of Structures (AoS).
     *
     * SoA is more cache-friendly when processing one field at a time.
     */
    public static function demonstrateLayouts(): void
    {
        // AoS: Array of Structures (common but less cache-friendly)
        $users = [
            ['name' => 'Alice', 'age' => 30, 'score' => 95],
            ['name' => 'Bob', 'age' => 25, 'score' => 87],
            // Each row mixes different types in memory
        ];

        // SoA: Structure of Arrays (more cache-friendly for columnar access)
        $names = ['Alice', 'Bob'];
        $ages = [30, 25];
        $scores = [95, 87];
        // Processing all scores touches contiguous memory
    }
}

package cache

import "math/rand/v2"

// ProcessSequential demonstrates cache-friendly sequential access.
// Go slices are contiguous in memory, making sequential traversal very fast.
func ProcessSequential(data []float64) float64 {
	sum := 0.0
	for _, v := range data {
		sum += v
	}
	return sum
}

// ProcessRandom demonstrates cache-unfriendly random access.
// Each access likely causes a cache miss.
func ProcessRandom(data []float64) float64 {
	sum := 0.0
	indices := make([]int, len(data))
	for i := range indices {
		indices[i] = i
	}
	rand.Shuffle(len(indices), func(i, j int) {
		indices[i], indices[j] = indices[j], indices[i]
	})

	for _, idx := range indices {
		sum += data[idx]
	}
	return sum
}

// AoS: Array of Structures (common but less cache-friendly for columnar access)
type User struct {
	Name  string
	Age   int
	Score int
}

// SoA: Structure of Arrays (more cache-friendly when processing one field)
type Users struct {
	Names  []string
	Ages   []int
	Scores []int
}

// SumScoresSoA processes all scores touching contiguous memory.
func SumScoresSoA(u Users) int {
	sum := 0
	for _, s := range u.Scores {
		sum += s
	}
	return sum
}

namespace Foundations.Caching;

/// <summary>
/// Cache-friendly data access patterns.
/// Sequential access is much faster than random due to CPU cache behavior.
/// </summary>
public static class CacheAwareProcessing
{
    /// <summary>
    /// Sequential access: cache-friendly.
    /// CPU prefetcher predicts the pattern and preloads data.
    /// </summary>
    public static double ProcessSequential(double[] data)
    {
        var sum = 0.0;
        for (var i = 0; i < data.Length; i++)
        {
            sum += data[i];
        }

        return sum;
    }

    /// <summary>
    /// Random access: cache-unfriendly.
    /// Each access likely causes a cache miss.
    /// </summary>
    public static double ProcessRandom(double[] data)
    {
        var indices = Enumerable.Range(0, data.Length).ToArray();
        Random.Shared.Shuffle(indices);

        var sum = 0.0;
        foreach (var index in indices)
        {
            sum += data[index];
        }

        return sum;
    }
}

// AoS: Array of Structures (common but less cache-friendly for columnar access)
public readonly record struct User(string Name, int Age, int Score);

// SoA: Structure of Arrays (more cache-friendly when processing one field)
public sealed class Users
{
    public required string[] Names { get; init; }
    public required int[] Ages { get; init; }
    public required int[] Scores { get; init; }
}

public static class ScoreAggregator
{
    // Processing all scores touches contiguous memory
    public static int SumScoresSoA(Users users) => users.Scores.Sum();
}

from __future__ import annotations

import random
from dataclasses import dataclass, field

import numpy as np
from numpy.typing import NDArray

def process_sequential(data: NDArray[np.float64]) -> float:
    """Sequential access: cache-friendly.

    A NumPy array is a contiguous C buffer, so the CPU prefetcher
    predicts the pattern and preloads data. A plain Python list holds
    pointers to boxed floats, which defeats the prefetcher.
    """
    return float(data.sum())

def process_random(data: NDArray[np.float64]) -> float:
    """Random access: cache-unfriendly. Each access likely causes a cache miss."""
    indices = list(range(len(data)))
    random.shuffle(indices)

    return float(sum(data[i] for i in indices))

# AoS: Array of Structures (common but less cache-friendly for columnar access)
@dataclass(slots=True)
class User:
    name: str
    age: int
    score: int

# SoA: Structure of Arrays (more cache-friendly when processing one field)
@dataclass(slots=True)
class Users:
    names: list[str] = field(default_factory=list)
    ages: NDArray[np.int64] = field(default_factory=lambda: np.empty(0, dtype=np.int64))
    scores: NDArray[np.int64] = field(default_factory=lambda: np.empty(0, dtype=np.int64))

def sum_scores_soa(users: Users) -> int:
    """Processing all scores touches contiguous memory."""
    return int(users.scores.sum())

<?php

declare(strict_types=1);

/**
 * Back-of-the-envelope estimation helper.
 * Essential skill for System Design interviews.
 */
final class CapacityEstimator
{
    private const SECONDS_PER_DAY = 86_400;
    private const SECONDS_PER_MONTH = 2_592_000; // 30 days
    private const SECONDS_PER_YEAR = 31_536_000; // 365 days

    /**
     * Estimate storage requirements.
     *
     * @param int $dailyActiveUsers DAU
     * @param int $actionsPerUser Average actions per user per day
     * @param int $bytesPerAction Average bytes per action (record)
     * @param int $retentionYears How many years to keep data
     * @return array{daily_gb: float, monthly_gb: float, total_tb: float}
     */
    public static function estimateStorage(
        int $dailyActiveUsers,
        int $actionsPerUser,
        int $bytesPerAction,
        int $retentionYears = 3,
    ): array {
        $dailyActions = $dailyActiveUsers * $actionsPerUser;
        $dailyBytes = $dailyActions * $bytesPerAction;

        $dailyGb = $dailyBytes / (1024 ** 3);
        $monthlyGb = $dailyGb * 30;
        $totalTb = ($dailyGb * 365 * $retentionYears) / 1024;

        return [
            'daily_gb' => round($dailyGb, 2),
            'monthly_gb' => round($monthlyGb, 2),
            'total_tb' => round($totalTb, 2),
        ];
    }

    /**
     * Estimate required throughput.
     *
     * @return array{avg_rps: int, peak_rps: int, bandwidth_mbps: float}
     */
    public static function estimateThroughput(
        int $dailyActiveUsers,
        int $actionsPerUser,
        int $bytesPerAction,
        float $peakMultiplier = 3.0,
    ): array {
        $totalDaily = $dailyActiveUsers * $actionsPerUser;
        $avgRps = (int) ceil($totalDaily / self::SECONDS_PER_DAY);
        $peakRps = (int) ceil($avgRps * $peakMultiplier);
        $bandwidthMbps = ($peakRps * $bytesPerAction * 8) / (1024 * 1024);

        return [
            'avg_rps' => $avgRps,
            'peak_rps' => $peakRps,
            'bandwidth_mbps' => round($bandwidthMbps, 2),
        ];
    }
}

// Example: Social media platform
// 10M DAU, 20 posts/views per day, 2KB per action, 3 year retention
$storage = CapacityEstimator::estimateStorage(
    dailyActiveUsers: 10_000_000,
    actionsPerUser: 20,
    bytesPerAction: 2048,
    retentionYears: 3,
);
// daily: ~0.37 GB, monthly: ~11.18 GB, total: ~1.31 TB

$throughput = CapacityEstimator::estimateThroughput(
    dailyActiveUsers: 10_000_000,
    actionsPerUser: 20,
    bytesPerAction: 2048,
);
// avg: ~2315 RPS, peak: ~6944 RPS

package capacity

import "math"

const (
	secondsPerDay = 86_400
)

// StorageEstimate holds storage capacity estimation results.
type StorageEstimate struct {
	DailyGB  float64
	MonthlyGB float64
	TotalTB  float64
}

// ThroughputEstimate holds throughput estimation results.
type ThroughputEstimate struct {
	AvgRPS       int
	PeakRPS      int
	BandwidthMbps float64
}

// EstimateStorage calculates storage requirements for capacity planning.
func EstimateStorage(dau, actionsPerUser, bytesPerAction, retentionYears int) StorageEstimate {
	dailyBytes := float64(dau) * float64(actionsPerUser) * float64(bytesPerAction)
	dailyGB := dailyBytes / (1024 * 1024 * 1024)
	monthlyGB := dailyGB * 30
	totalTB := (dailyGB * 365 * float64(retentionYears)) / 1024

	return StorageEstimate{
		DailyGB:   math.Round(dailyGB*100) / 100,
		MonthlyGB: math.Round(monthlyGB*100) / 100,
		TotalTB:   math.Round(totalTB*100) / 100,
	}
}

// EstimateThroughput calculates required request throughput.
func EstimateThroughput(dau, actionsPerUser, bytesPerAction int, peakMultiplier float64) ThroughputEstimate {
	totalDaily := float64(dau) * float64(actionsPerUser)
	avgRPS := int(math.Ceil(totalDaily / secondsPerDay))
	peakRPS := int(math.Ceil(float64(avgRPS) * peakMultiplier))
	bandwidthMbps := float64(peakRPS) * float64(bytesPerAction) * 8 / (1024 * 1024)

	return ThroughputEstimate{
		AvgRPS:        avgRPS,
		PeakRPS:       peakRPS,
		BandwidthMbps: math.Round(bandwidthMbps*100) / 100,
	}
}

// Example: Social media platform
// 10M DAU, 20 posts/views per day, 2KB per action, 3 year retention
//
// storage := EstimateStorage(10_000_000, 20, 2048, 3)
// daily: ~0.37 GB, monthly: ~11.18 GB, total: ~1.31 TB
//
// throughput := EstimateThroughput(10_000_000, 20, 2048, 3.0)
// avg: ~2315 RPS, peak: ~6944 RPS

namespace Foundations.Capacity;

public readonly record struct StorageEstimate(double DailyGb, double MonthlyGb, double TotalTb);

public readonly record struct ThroughputEstimate(int AvgRps, int PeakRps, double BandwidthMbps);

/// <summary>
/// Back-of-the-envelope estimation helper.
/// Essential skill for System Design interviews.
/// </summary>
public static class CapacityEstimator
{
    private const int SecondsPerDay = 86_400;
    private const double BytesPerGb = 1024d * 1024d * 1024d;

    /// <summary>Estimate storage requirements.</summary>
    public static StorageEstimate EstimateStorage(
        int dailyActiveUsers,
        int actionsPerUser,
        int bytesPerAction,
        int retentionYears = 3)
    {
        var dailyBytes = (double)dailyActiveUsers * actionsPerUser * bytesPerAction;
        var dailyGb = dailyBytes / BytesPerGb;
        var monthlyGb = dailyGb * 30;
        var totalTb = dailyGb * 365 * retentionYears / 1024;

        return new StorageEstimate(
            Math.Round(dailyGb, 2),
            Math.Round(monthlyGb, 2),
            Math.Round(totalTb, 2));
    }

    /// <summary>Estimate required throughput.</summary>
    public static ThroughputEstimate EstimateThroughput(
        int dailyActiveUsers,
        int actionsPerUser,
        int bytesPerAction,
        double peakMultiplier = 3.0)
    {
        var totalDaily = (double)dailyActiveUsers * actionsPerUser;
        var avgRps = (int)Math.Ceiling(totalDaily / SecondsPerDay);
        var peakRps = (int)Math.Ceiling(avgRps * peakMultiplier);
        var bandwidthMbps = (double)peakRps * bytesPerAction * 8 / (1024 * 1024);

        return new ThroughputEstimate(avgRps, peakRps, Math.Round(bandwidthMbps, 2));
    }
}

// Example: Social media platform
// 10M DAU, 20 posts/views per day, 2KB per action, 3 year retention
var storage = CapacityEstimator.EstimateStorage(
    dailyActiveUsers: 10_000_000,
    actionsPerUser: 20,
    bytesPerAction: 2048,
    retentionYears: 3);
// daily: ~0.37 GB, monthly: ~11.18 GB, total: ~1.31 TB

var throughput = CapacityEstimator.EstimateThroughput(
    dailyActiveUsers: 10_000_000,
    actionsPerUser: 20,
    bytesPerAction: 2048);
// avg: ~2315 RPS, peak: ~6944 RPS

from __future__ import annotations

import math
from typing import NamedTuple

SECONDS_PER_DAY = 86_400
BYTES_PER_GB = 1024**3

class StorageEstimate(NamedTuple):
    daily_gb: float
    monthly_gb: float
    total_tb: float

class ThroughputEstimate(NamedTuple):
    avg_rps: int
    peak_rps: int
    bandwidth_mbps: float

def estimate_storage(
    daily_active_users: int,
    actions_per_user: int,
    bytes_per_action: int,
    retention_years: int = 3,
) -> StorageEstimate:
    """Estimate storage requirements for capacity planning."""
    daily_bytes = daily_active_users * actions_per_user * bytes_per_action
    daily_gb = daily_bytes / BYTES_PER_GB
    monthly_gb = daily_gb * 30
    total_tb = daily_gb * 365 * retention_years / 1024

    return StorageEstimate(
        daily_gb=round(daily_gb, 2),
        monthly_gb=round(monthly_gb, 2),
        total_tb=round(total_tb, 2),
    )

def estimate_throughput(
    daily_active_users: int,
    actions_per_user: int,
    bytes_per_action: int,
    peak_multiplier: float = 3.0,
) -> ThroughputEstimate:
    """Estimate required request throughput and bandwidth."""
    total_daily = daily_active_users * actions_per_user
    avg_rps = math.ceil(total_daily / SECONDS_PER_DAY)
    peak_rps = math.ceil(avg_rps * peak_multiplier)
    bandwidth_mbps = peak_rps * bytes_per_action * 8 / (1024 * 1024)

    return ThroughputEstimate(
        avg_rps=avg_rps,
        peak_rps=peak_rps,
        bandwidth_mbps=round(bandwidth_mbps, 2),
    )

# Example: Social media platform
# 10M DAU, 20 posts/views per day, 2KB per action, 3 year retention
storage = estimate_storage(
    daily_active_users=10_000_000,
    actions_per_user=20,
    bytes_per_action=2048,
    retention_years=3,
)
# daily: ~0.37 GB, monthly: ~11.18 GB, total: ~1.31 TB

throughput = estimate_throughput(
    daily_active_users=10_000_000,
    actions_per_user=20,
    bytes_per_action=2048,
)
# avg: ~2315 RPS, peak: ~6944 RPS