Skip to main content

hydro_deploy/
aws.rs

1use std::any::Any;
2use std::borrow::Cow;
3use std::fmt::Debug;
4use std::sync::{Arc, Mutex, OnceLock};
5
6use anyhow::Result;
7use nanoid::nanoid;
8use serde_json::json;
9
10use super::terraform::{TERRAFORM_ALPHABET, TerraformOutput, TerraformProvider};
11use super::{ClientStrategy, Host, HostTargetType, LaunchedHost, ResourceBatch, ResourceResult};
12use crate::ssh::LaunchedSshHost;
13use crate::{BaseServerStrategy, HostStrategyGetter, PortNetworkHint};
14
15pub struct LaunchedEc2Instance {
16    resource_result: Arc<ResourceResult>,
17    user: String,
18    pub internal_ip: String,
19    pub external_ip: Option<String>,
20}
21
22impl LaunchedSshHost for LaunchedEc2Instance {
23    fn get_external_ip(&self) -> Option<&str> {
24        self.external_ip.as_deref()
25    }
26
27    fn get_internal_ip(&self) -> &str {
28        &self.internal_ip
29    }
30
31    fn get_cloud_provider(&self) -> &'static str {
32        "AWS"
33    }
34
35    fn resource_result(&self) -> &Arc<ResourceResult> {
36        &self.resource_result
37    }
38
39    fn ssh_user(&self) -> &str {
40        self.user.as_str()
41    }
42}
43
44#[derive(Debug, Clone)]
45pub struct NetworkResources {
46    vpc: String,
47    subnet: String,
48    security_group: String,
49}
50
51#[derive(Debug)]
52pub struct AwsNetwork {
53    pub region: String,
54    pub existing_network_key: OnceLock<NetworkResources>,
55    pub existing_network_id: OnceLock<NetworkResources>,
56    id: String,
57}
58
59impl AwsNetwork {
60    pub fn new(region: impl Into<String>, existing_vpc: Option<NetworkResources>) -> Arc<Self> {
61        Arc::new(Self {
62            region: region.into(),
63            existing_network_key: OnceLock::new(),
64            existing_network_id: existing_vpc.map(From::from).unwrap_or_default(),
65            id: nanoid!(8, &TERRAFORM_ALPHABET),
66        })
67    }
68
69    fn collect_resources(&self, resource_batch: &mut ResourceBatch) -> NetworkResources {
70        resource_batch
71            .terraform
72            .terraform
73            .required_providers
74            .insert(
75                "aws".to_owned(),
76                TerraformProvider {
77                    source: "hashicorp/aws".to_owned(),
78                    version: "5.0.0".to_owned(),
79                },
80            );
81
82        resource_batch.terraform.provider.insert(
83            "aws".to_owned(),
84            json!({
85                "region": self.region
86            }),
87        );
88
89        let vpc_network = format!("hydro-vpc-network-{}", self.id);
90        let subnet_key = format!("{vpc_network}-subnet");
91        let sg_key = format!("{vpc_network}-default-sg");
92
93        if let Some(existing) = self.existing_network_id.get() {
94            let mut resolve = |resource_type: &str, existing_id: &str, data_key: String| {
95                resource_batch
96                    .terraform
97                    .data
98                    .entry(resource_type.to_owned())
99                    .or_default()
100                    .insert(data_key.clone(), json!({ "id": existing_id }));
101                format!("data.{resource_type}.{data_key}")
102            };
103
104            NetworkResources {
105                vpc: resolve("aws_vpc", &existing.vpc, vpc_network),
106                subnet: resolve("aws_subnet", &existing.subnet, subnet_key),
107                security_group: resolve("aws_security_group", &existing.security_group, sg_key),
108            }
109        } else if let Some(existing) = self.existing_network_key.get() {
110            NetworkResources {
111                vpc: format!("aws_vpc.{}", existing.vpc),
112                subnet: format!("aws_subnet.{}", existing.subnet),
113                security_group: format!("aws_security_group.{}", existing.security_group),
114            }
115        } else {
116            resource_batch
117                .terraform
118                .resource
119                .entry("aws_vpc".to_owned())
120                .or_default()
121                .insert(
122                    vpc_network.clone(),
123                    json!({
124                        "cidr_block": "10.0.0.0/16",
125                        "enable_dns_hostnames": true,
126                        "enable_dns_support": true,
127                        "tags": {
128                            "Name": vpc_network
129                        }
130                    }),
131                );
132
133            // Create internet gateway
134            let igw_key = format!("{vpc_network}-igw");
135            resource_batch
136                .terraform
137                .resource
138                .entry("aws_internet_gateway".to_owned())
139                .or_default()
140                .insert(
141                    igw_key.clone(),
142                    json!({
143                        "vpc_id": format!("${{aws_vpc.{}.id}}", vpc_network),
144                        "tags": {
145                            "Name": igw_key
146                        }
147                    }),
148                );
149
150            // Create subnet
151            resource_batch
152                .terraform
153                .resource
154                .entry("aws_subnet".to_owned())
155                .or_default()
156                .insert(
157                    subnet_key.clone(),
158                    json!({
159                        "vpc_id": format!("${{aws_vpc.{}.id}}", vpc_network),
160                        "cidr_block": "10.0.1.0/24",
161                        "availability_zone": format!("{}a", self.region),
162                        "map_public_ip_on_launch": true,
163                        "tags": {
164                            "Name": subnet_key
165                        }
166                    }),
167                );
168
169            // Create route table
170            let rt_key = format!("{vpc_network}-rt");
171            resource_batch
172                .terraform
173                .resource
174                .entry("aws_route_table".to_owned())
175                .or_default()
176                .insert(
177                    rt_key.clone(),
178                    json!({
179                        "vpc_id": format!("${{aws_vpc.{}.id}}", vpc_network),
180                        "tags": {
181                            "Name": rt_key
182                        }
183                    }),
184                );
185
186            // Create route
187            resource_batch
188                .terraform
189                .resource
190                .entry("aws_route".to_owned())
191                .or_default()
192                .insert(
193                    format!("{vpc_network}-route"),
194                    json!({
195                        "route_table_id": format!("${{aws_route_table.{}.id}}", rt_key),
196                        "destination_cidr_block": "0.0.0.0/0",
197                        "gateway_id": format!("${{aws_internet_gateway.{}.id}}", igw_key)
198                    }),
199                );
200
201            resource_batch
202                .terraform
203                .resource
204                .entry("aws_route_table_association".to_owned())
205                .or_default()
206                .insert(
207                    format!("{vpc_network}-rta"),
208                    json!({
209                        "subnet_id": format!("${{aws_subnet.{}.id}}", subnet_key),
210                        "route_table_id": format!("${{aws_route_table.{}.id}}", rt_key)
211                    }),
212                );
213
214            // Create security group that allows internal communication
215            resource_batch
216                .terraform
217                .resource
218                .entry("aws_security_group".to_owned())
219                .or_default()
220                .insert(
221                    sg_key.clone(),
222                    json!({
223                        "name": format!("{vpc_network}-default-allow-internal"),
224                        "description": "Allow internal communication between instances",
225                        "vpc_id": format!("${{aws_vpc.{}.id}}", vpc_network),
226                        "ingress": [
227                            {
228                                "from_port": 0,
229                                "to_port": 65535,
230                                "protocol": "tcp",
231                                "cidr_blocks": ["10.0.0.0/16"],
232                                "description": "Allow all TCP traffic within VPC",
233                                "ipv6_cidr_blocks": [],
234                                "prefix_list_ids": [],
235                                "security_groups": [],
236                                "self": false
237                            },
238                            {
239                                "from_port": 0,
240                                "to_port": 65535,
241                                "protocol": "udp",
242                                "cidr_blocks": ["10.0.0.0/16"],
243                                "description": "Allow all UDP traffic within VPC",
244                                "ipv6_cidr_blocks": [],
245                                "prefix_list_ids": [],
246                                "security_groups": [],
247                                "self": false
248                            },
249                            {
250                                "from_port": -1,
251                                "to_port": -1,
252                                "protocol": "icmp",
253                                "cidr_blocks": ["10.0.0.0/16"],
254                                "description": "Allow ICMP within VPC",
255                                "ipv6_cidr_blocks": [],
256                                "prefix_list_ids": [],
257                                "security_groups": [],
258                                "self": false
259                            }
260                        ],
261                        "egress": [
262                            {
263                                "from_port": 0,
264                                "to_port": 0,
265                                "protocol": "-1",
266                                "cidr_blocks": ["0.0.0.0/0"],
267                                "description": "Allow all outbound traffic",
268                                "ipv6_cidr_blocks": [],
269                                "prefix_list_ids": [],
270                                "security_groups": [],
271                                "self": false
272                            }
273                        ]
274                    }),
275                );
276
277            let resources = NetworkResources {
278                vpc: format!("aws_vpc.{vpc_network}"),
279                subnet: format!("aws_subnet.{subnet_key}"),
280                security_group: format!("aws_security_group.{sg_key}"),
281            };
282
283            // Add outputs so we can retrieve actual AWS IDs after apply
284            resource_batch.terraform.output.insert(
285                format!("hydro-network-{}-vpc-id", self.id),
286                TerraformOutput {
287                    value: format!("${{aws_vpc.{vpc_network}.id}}"),
288                },
289            );
290            resource_batch.terraform.output.insert(
291                format!("hydro-network-{}-subnet-id", self.id),
292                TerraformOutput {
293                    value: format!("${{aws_subnet.{subnet_key}.id}}"),
294                },
295            );
296            resource_batch.terraform.output.insert(
297                format!("hydro-network-{}-sg-id", self.id),
298                TerraformOutput {
299                    value: format!("${{aws_security_group.{sg_key}.id}}"),
300                },
301            );
302
303            let _ = self.existing_network_key.set(NetworkResources {
304                vpc: vpc_network,
305                subnet: subnet_key,
306                security_group: sg_key,
307            });
308            resources
309        }
310    }
311
312    pub fn update_from_outputs(&self, resource_result: &ResourceResult) {
313        let outputs = &resource_result.terraform.outputs;
314        if let (Some(vpc), Some(subnet), Some(sg)) = (
315            outputs.get(&format!("hydro-network-{}-vpc-id", self.id)),
316            outputs.get(&format!("hydro-network-{}-subnet-id", self.id)),
317            outputs.get(&format!("hydro-network-{}-sg-id", self.id)),
318        ) {
319            let _ = self.existing_network_id.set(NetworkResources {
320                vpc: vpc.value.clone(),
321                subnet: subnet.value.clone(),
322                security_group: sg.value.clone(),
323            });
324        }
325    }
326}
327
328/// Represents a IAM role, IAM policy attachments, and instance profile for one or multiple EC2 instances.
329#[derive(Debug)]
330pub struct AwsEc2IamInstanceProfile {
331    pub region: String,
332    pub existing_instance_profile_key_or_name: Option<String>,
333    pub policy_arns: Vec<String>,
334    id: String,
335}
336
337impl AwsEc2IamInstanceProfile {
338    /// Creates a new instance. If `existing_instance_profile_name` is `Some`, that will be used as the instance
339    /// profile name which must already exist in the AWS account.
340    pub fn new(region: impl Into<String>, existing_instance_profile_name: Option<String>) -> Self {
341        Self {
342            region: region.into(),
343            existing_instance_profile_key_or_name: existing_instance_profile_name,
344            policy_arns: Default::default(),
345            id: nanoid!(8, &TERRAFORM_ALPHABET),
346        }
347    }
348
349    /// Permits the given ARN.
350    pub fn add_policy_arn(mut self, policy_arn: impl Into<String>) -> Self {
351        if self.existing_instance_profile_key_or_name.is_some() {
352            panic!("Adding an ARN to an existing instance profile is not supported.");
353        }
354        self.policy_arns.push(policy_arn.into());
355        self
356    }
357
358    /// Enables running and emitting telemetry via the CloudWatch agent.
359    pub fn add_cloudwatch_agent_server_policy_arn(self) -> Self {
360        self.add_policy_arn("arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy")
361    }
362
363    fn collect_resources(&mut self, resource_batch: &mut ResourceBatch) -> String {
364        const RESOURCE_AWS_IAM_INSTANCE_PROFILE: &str = "aws_iam_instance_profile";
365        const RESOURCE_AWS_IAM_ROLE_POLICY_ATTACHMENT: &str = "aws_iam_role_policy_attachment";
366        const RESOURCE_AWS_IAM_ROLE: &str = "aws_iam_role";
367
368        resource_batch
369            .terraform
370            .terraform
371            .required_providers
372            .insert(
373                "aws".to_owned(),
374                TerraformProvider {
375                    source: "hashicorp/aws".to_owned(),
376                    version: "5.0.0".to_owned(),
377                },
378            );
379
380        resource_batch.terraform.provider.insert(
381            "aws".to_owned(),
382            json!({
383                "region": self.region
384            }),
385        );
386
387        let instance_profile_key = format!("hydro-instance-profile-{}", self.id);
388
389        if let Some(existing) = self.existing_instance_profile_key_or_name.as_ref() {
390            if resource_batch
391                .terraform
392                .resource
393                .get(RESOURCE_AWS_IAM_INSTANCE_PROFILE)
394                .is_some_and(|map| map.contains_key(existing))
395            {
396                // `existing` is a key.
397                format!("{RESOURCE_AWS_IAM_INSTANCE_PROFILE}.{existing}")
398            } else {
399                // `existing` is a name of an existing resource, supplied when constructed.
400                resource_batch
401                    .terraform
402                    .data
403                    .entry(RESOURCE_AWS_IAM_INSTANCE_PROFILE.to_owned())
404                    .or_default()
405                    .insert(
406                        instance_profile_key.clone(),
407                        json!({
408                            "id": existing,
409                        }),
410                    );
411
412                format!("data.{RESOURCE_AWS_IAM_INSTANCE_PROFILE}.{instance_profile_key}")
413            }
414        } else {
415            // Create the role (permissions set after).
416            let iam_role_key = format!("{instance_profile_key}-iam-role");
417            resource_batch
418                .terraform
419                .resource
420                .entry(RESOURCE_AWS_IAM_ROLE.to_owned())
421                .or_default()
422                .insert(
423                    iam_role_key.clone(),
424                    json!({
425                        "name": format!("hydro-iam-role-{}", self.id),
426                        "assume_role_policy": json!({
427                            "Version": "2012-10-17",
428                            "Statement": [
429                                {
430                                    "Action": "sts:AssumeRole",
431                                    "Effect": "Allow",
432                                    "Principal": {
433                                        "Service": "ec2.amazonaws.com"
434                                    }
435                                }
436                            ]
437                        }).to_string(),
438                    }),
439                );
440
441            // Attach permissions
442            for (i, policy_arn) in self.policy_arns.iter().enumerate() {
443                let policy_attachment_key = format!("{iam_role_key}-policy-attachment-{i}");
444                resource_batch
445                    .terraform
446                    .resource
447                    .entry(RESOURCE_AWS_IAM_ROLE_POLICY_ATTACHMENT.to_owned())
448                    .or_default()
449                    .insert(
450                        policy_attachment_key,
451                        json!({
452                            "policy_arn": policy_arn,
453                            "role": format!("${{{RESOURCE_AWS_IAM_ROLE}.{iam_role_key}.name}}"),
454                        }),
455                    );
456            }
457
458            // Create instance profile. This is what attaches to EC2 instances.
459            resource_batch
460                .terraform
461                .resource
462                .entry(RESOURCE_AWS_IAM_INSTANCE_PROFILE.to_owned())
463                .or_default()
464                .insert(
465                    instance_profile_key.clone(),
466                    json!({
467                        "name": format!("hydro-instance-profile-{}", self.id),
468                        "role": format!("${{{RESOURCE_AWS_IAM_ROLE}.{iam_role_key}.name}}"),
469                    }),
470                );
471
472            // Set key
473            self.existing_instance_profile_key_or_name = Some(instance_profile_key.clone());
474
475            format!("{RESOURCE_AWS_IAM_INSTANCE_PROFILE}.{instance_profile_key}")
476        }
477    }
478}
479
480/// Represents a CloudWatch log group.
481#[derive(Debug)]
482pub struct AwsCloudwatchLogGroup {
483    pub region: String,
484    pub existing_cloudwatch_log_group_key_or_name: Option<String>,
485    id: String,
486}
487
488impl AwsCloudwatchLogGroup {
489    /// Creates a new instance. If `existing_cloudwatch_log_group_name` is `Some`, that will be used as the CloudWatch
490    /// log group name which must already exist in the AWS account and region.
491    pub fn new(
492        region: impl Into<String>,
493        existing_cloudwatch_log_group_name: Option<String>,
494    ) -> Self {
495        Self {
496            region: region.into(),
497            existing_cloudwatch_log_group_key_or_name: existing_cloudwatch_log_group_name,
498            id: nanoid!(8, &TERRAFORM_ALPHABET),
499        }
500    }
501
502    fn collect_resources(&mut self, resource_batch: &mut ResourceBatch) -> String {
503        const RESOURCE_AWS_CLOUDWATCH_LOG_GROUP: &str = "aws_cloudwatch_log_group";
504
505        resource_batch
506            .terraform
507            .terraform
508            .required_providers
509            .insert(
510                "aws".to_owned(),
511                TerraformProvider {
512                    source: "hashicorp/aws".to_owned(),
513                    version: "5.0.0".to_owned(),
514                },
515            );
516
517        resource_batch.terraform.provider.insert(
518            "aws".to_owned(),
519            json!({
520                "region": self.region
521            }),
522        );
523
524        let cloudwatch_log_group_key = format!("hydro-cloudwatch-log-group-{}", self.id);
525
526        if let Some(existing) = self.existing_cloudwatch_log_group_key_or_name.as_ref() {
527            if resource_batch
528                .terraform
529                .resource
530                .get(RESOURCE_AWS_CLOUDWATCH_LOG_GROUP)
531                .is_some_and(|map| map.contains_key(existing))
532            {
533                // `existing` is a key.
534                format!("{RESOURCE_AWS_CLOUDWATCH_LOG_GROUP}.{existing}")
535            } else {
536                // `existing` is a name of an existing resource, supplied when constructed.
537                resource_batch
538                    .terraform
539                    .data
540                    .entry(RESOURCE_AWS_CLOUDWATCH_LOG_GROUP.to_owned())
541                    .or_default()
542                    .insert(
543                        cloudwatch_log_group_key.clone(),
544                        json!({
545                            "id": existing,
546                        }),
547                    );
548
549                format!("data.{RESOURCE_AWS_CLOUDWATCH_LOG_GROUP}.{cloudwatch_log_group_key}")
550            }
551        } else {
552            // Create the log group.
553            resource_batch
554                .terraform
555                .resource
556                .entry(RESOURCE_AWS_CLOUDWATCH_LOG_GROUP.to_owned())
557                .or_default()
558                .insert(
559                    cloudwatch_log_group_key.clone(),
560                    json!({
561                        "name": format!("hydro-cloudwatch-log-group-{}", self.id),
562                        "retention_in_days": 1,
563                    }),
564                );
565
566            // Set key
567            self.existing_cloudwatch_log_group_key_or_name = Some(cloudwatch_log_group_key.clone());
568
569            format!("{RESOURCE_AWS_CLOUDWATCH_LOG_GROUP}.{cloudwatch_log_group_key}")
570        }
571    }
572}
573
574pub struct AwsEc2Host {
575    /// ID from [`crate::Deployment::add_host`].
576    id: usize,
577
578    region: String,
579    instance_type: String,
580    target_type: HostTargetType,
581    ami: String,
582    network: Arc<AwsNetwork>,
583    iam_instance_profile: Option<Arc<Mutex<AwsEc2IamInstanceProfile>>>,
584    cloudwatch_log_group: Option<Arc<Mutex<AwsCloudwatchLogGroup>>>,
585    cwa_metrics_collected: Option<serde_json::Value>,
586    user: Option<String>,
587    display_name: Option<String>,
588    pub launched: OnceLock<Arc<LaunchedEc2Instance>>,
589    external_ports: Mutex<Vec<u16>>,
590}
591
592impl Debug for AwsEc2Host {
593    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
594        f.write_fmt(format_args!(
595            "AwsEc2Host({} ({:?}))",
596            self.id, self.display_name,
597        ))
598    }
599}
600
601impl AwsEc2Host {
602    #[expect(clippy::too_many_arguments, reason = "used via builder pattern")]
603    pub fn new(
604        id: usize,
605        region: impl Into<String>,
606        instance_type: impl Into<String>,
607        target_type: HostTargetType,
608        ami: impl Into<String>,
609        network: Arc<AwsNetwork>,
610        iam_instance_profile: Option<Arc<Mutex<AwsEc2IamInstanceProfile>>>,
611        cloudwatch_log_group: Option<Arc<Mutex<AwsCloudwatchLogGroup>>>,
612        cwa_metrics_collected: Option<serde_json::Value>,
613        user: Option<String>,
614        display_name: Option<String>,
615    ) -> Self {
616        Self {
617            id,
618            region: region.into(),
619            instance_type: instance_type.into(),
620            target_type,
621            ami: ami.into(),
622            network,
623            iam_instance_profile,
624            cloudwatch_log_group,
625            cwa_metrics_collected,
626            user,
627            display_name,
628            launched: OnceLock::new(),
629            external_ports: Mutex::new(Vec::new()),
630        }
631    }
632}
633
634impl Host for AwsEc2Host {
635    fn target_type(&self) -> HostTargetType {
636        self.target_type
637    }
638
639    fn request_port_base(&self, bind_type: &BaseServerStrategy) {
640        match bind_type {
641            BaseServerStrategy::UnixSocket => {}
642            BaseServerStrategy::InternalTcpPort(_) => {}
643            BaseServerStrategy::ExternalTcpPort(port) => {
644                let mut external_ports = self.external_ports.lock().unwrap();
645                if !external_ports.contains(port) {
646                    if self.launched.get().is_some() {
647                        todo!("Cannot adjust security group after host has been launched");
648                    }
649                    external_ports.push(*port);
650                }
651            }
652        }
653    }
654
655    fn request_custom_binary(&self) {
656        self.request_port_base(&BaseServerStrategy::ExternalTcpPort(22));
657    }
658
659    fn id(&self) -> usize {
660        self.id
661    }
662
663    fn collect_resources(&self, resource_batch: &mut ResourceBatch) {
664        if self.launched.get().is_some() {
665            return;
666        }
667
668        let network_resources = self.network.collect_resources(resource_batch);
669
670        let iam_instance_profile = self
671            .iam_instance_profile
672            .as_deref()
673            .map(|irip| irip.lock().unwrap().collect_resources(resource_batch));
674
675        let cloudwatch_log_group = self
676            .cloudwatch_log_group
677            .as_deref()
678            .map(|cwlg| cwlg.lock().unwrap().collect_resources(resource_batch));
679
680        // Add additional providers
681        resource_batch
682            .terraform
683            .terraform
684            .required_providers
685            .insert(
686                "local".to_owned(),
687                TerraformProvider {
688                    source: "hashicorp/local".to_owned(),
689                    version: "2.3.0".to_owned(),
690                },
691            );
692
693        resource_batch
694            .terraform
695            .terraform
696            .required_providers
697            .insert(
698                "tls".to_owned(),
699                TerraformProvider {
700                    source: "hashicorp/tls".to_owned(),
701                    version: "4.0.4".to_owned(),
702                },
703            );
704
705        // Generate SSH key pair
706        resource_batch
707            .terraform
708            .resource
709            .entry("tls_private_key".to_owned())
710            .or_default()
711            .insert(
712                "vm_instance_ssh_key".to_owned(),
713                json!({
714                    "algorithm": "RSA",
715                    "rsa_bits": 4096
716                }),
717            );
718
719        resource_batch
720            .terraform
721            .resource
722            .entry("local_file".to_owned())
723            .or_default()
724            .insert(
725                "vm_instance_ssh_key_pem".to_owned(),
726                json!({
727                    "content": "${tls_private_key.vm_instance_ssh_key.private_key_pem}",
728                    "filename": ".ssh/vm_instance_ssh_key_pem",
729                    "file_permission": "0600",
730                    "directory_permission": "0700"
731                }),
732            );
733
734        resource_batch
735            .terraform
736            .resource
737            .entry("aws_key_pair".to_owned())
738            .or_default()
739            .insert(
740                "ec2_key_pair".to_owned(),
741                json!({
742                    "key_name": format!("hydro-key-{}", nanoid!(8, &TERRAFORM_ALPHABET)),
743                    "public_key": "${tls_private_key.vm_instance_ssh_key.public_key_openssh}"
744                }),
745            );
746
747        let instance_key = format!("ec2-instance-{}", self.id);
748        let mut instance_name = format!("hydro-ec2-instance-{}", nanoid!(8, &TERRAFORM_ALPHABET));
749
750        if let Some(mut display_name) = self.display_name.clone() {
751            instance_name.push('-');
752            display_name = display_name.replace("_", "-").to_lowercase();
753
754            let num_chars_to_cut = instance_name.len() + display_name.len() - 63;
755            if num_chars_to_cut > 0 {
756                display_name.drain(0..num_chars_to_cut);
757            }
758            instance_name.push_str(&display_name);
759        }
760
761        let vpc_ref = format!("${{{}.id}}", network_resources.vpc);
762        let default_sg_ref = format!("${{{}.id}}", network_resources.security_group);
763
764        // Create additional security group for external ports if needed
765        let mut security_groups = vec![default_sg_ref];
766        let external_ports = self.external_ports.lock().unwrap();
767
768        if !external_ports.is_empty() {
769            let sg_key = format!("sg-{}", self.id);
770            let mut sg_rules = vec![];
771
772            for port in external_ports.iter() {
773                sg_rules.push(json!({
774                    "from_port": port,
775                    "to_port": port,
776                    "protocol": "tcp",
777                    "cidr_blocks": ["0.0.0.0/0"],
778                    "description": format!("External port {}", port),
779                    "ipv6_cidr_blocks": [],
780                    "prefix_list_ids": [],
781                    "security_groups": [],
782                    "self": false
783                }));
784            }
785
786            resource_batch
787                .terraform
788                .resource
789                .entry("aws_security_group".to_owned())
790                .or_default()
791                .insert(
792                    sg_key.clone(),
793                    json!({
794                        "name": format!("hydro-sg-{}", nanoid!(8, &TERRAFORM_ALPHABET)),
795                        "description": "Hydro external ports security group",
796                        "vpc_id": vpc_ref,
797                        "ingress": sg_rules,
798                        "egress": [{
799                            "from_port": 0,
800                            "to_port": 0,
801                            "protocol": "-1",
802                            "cidr_blocks": ["0.0.0.0/0"],
803                            "description": "All outbound traffic",
804                            "ipv6_cidr_blocks": [],
805                            "prefix_list_ids": [],
806                            "security_groups": [],
807                            "self": false
808                        }]
809                    }),
810                );
811
812            security_groups.push(format!("${{aws_security_group.{}.id}}", sg_key));
813        }
814        drop(external_ports);
815
816        let subnet_ref = format!("${{{}.id}}", network_resources.subnet);
817        let iam_instance_profile_ref = iam_instance_profile.map(|key| format!("${{{key}.name}}"));
818
819        // Write the CloudWatch Agent config file.
820        // https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html
821        let cloudwatch_agent_config = cloudwatch_log_group.map(|cwlg| {
822            json!({
823                "logs": {
824                    "logs_collected": {
825                        "files": {
826                            "collect_list": [
827                                {
828                                    "file_path": "/var/log/hydro/metrics.log",
829                                    "log_group_name": format!("${{{cwlg}.name}}"), // This `$` is interpreted by terraform
830                                    "log_stream_name": "{{instance_id}}"
831                                }
832                            ]
833                        }
834                    }
835                },
836                "metrics": {
837                    // "namespace": todo!(), // TODO(mingwei): use flow_name here somehow
838                    "metrics_collected": self.cwa_metrics_collected.as_ref().map(Cow::Borrowed).unwrap_or_else(|| Cow::Owned(json!({
839                        "cpu": {
840                            "resources": [
841                                "*"
842                            ],
843                            "measurement": [
844                                "usage_active"
845                            ],
846                            "totalcpu": true
847                        },
848                        "mem": {
849                            "measurement": [
850                                "used_percent"
851                            ]
852                        }
853                    }))),
854                    // See special escape handling below.
855                    "append_dimensions": {
856                        "InstanceId": "${aws:InstanceId}"
857                    }
858                }
859            })
860            .to_string()
861        });
862
863        // TODO(mingwei): Run this in SSH instead of `user_data` to avoid racing and capture errors.
864        let user_data_script = cloudwatch_agent_config.map(|cwa_config| {
865            let cwa_config_esc = cwa_config
866                .replace("\\", r"\\") // escape backslashes
867                .replace("\"", r#"\""#) // escape quotes
868                .replace("\n", r"\n") // escape newlines
869                // Special handling of AWS `append_dimensions` fields:
870                // `$$` to escape for terraform, becomes `\$` in bash, becomes `$` in echo output.
871                .replace("${aws:", r"\$${aws:");
872            format!(
873                r##"
874#!/bin/bash
875set -euxo pipefail
876
877mkdir -p /var/log/hydro/
878chmod +777 /var/log/hydro
879touch /var/log/hydro/metrics.log
880chmod +666 /var/log/hydro/metrics.log
881
882# Install the CloudWatch Agent
883yum install -y amazon-cloudwatch-agent
884
885mkdir -p /opt/aws/amazon-cloudwatch-agent/etc
886echo -e "{cwa_config_esc}" > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
887
888# Start or restart the agent
889/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
890    -a fetch-config -m ec2 \
891    -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \
892    -s
893"##
894            )
895        });
896
897        // Create EC2 instance
898        resource_batch
899            .terraform
900            .resource
901            .entry("aws_instance".to_owned())
902            .or_default()
903            .insert(
904                instance_key.clone(),
905                json!({
906                    "ami": self.ami,
907                    "instance_type": self.instance_type,
908                    "key_name": "${aws_key_pair.ec2_key_pair.key_name}",
909                    "vpc_security_group_ids": security_groups,
910                    "subnet_id": subnet_ref,
911                    "associate_public_ip_address": true,
912                    "iam_instance_profile": iam_instance_profile_ref, // May be `None`.
913                    "user_data": user_data_script, // May be `None`.
914                    "tags": {
915                        "Name": instance_name
916                    }
917                }),
918            );
919
920        resource_batch.terraform.output.insert(
921            format!("{}-private-ip", instance_key),
922            TerraformOutput {
923                value: format!("${{aws_instance.{}.private_ip}}", instance_key),
924            },
925        );
926
927        resource_batch.terraform.output.insert(
928            format!("{}-public-ip", instance_key),
929            TerraformOutput {
930                value: format!("${{aws_instance.{}.public_ip}}", instance_key),
931            },
932        );
933    }
934
935    fn launched(&self) -> Option<Arc<dyn LaunchedHost>> {
936        self.launched
937            .get()
938            .map(|a| a.clone() as Arc<dyn LaunchedHost>)
939    }
940
941    fn provision(&self, resource_result: &Arc<ResourceResult>) -> Arc<dyn LaunchedHost> {
942        self.launched
943            .get_or_init(|| {
944                let id = self.id;
945
946                self.network.update_from_outputs(resource_result);
947                let internal_ip = resource_result
948                    .terraform
949                    .outputs
950                    .get(&format!("ec2-instance-{id}-private-ip"))
951                    .unwrap()
952                    .value
953                    .clone();
954
955                let external_ip = resource_result
956                    .terraform
957                    .outputs
958                    .get(&format!("ec2-instance-{id}-public-ip"))
959                    .map(|v| v.value.clone());
960
961                Arc::new(LaunchedEc2Instance {
962                    resource_result: resource_result.clone(),
963                    user: self.user.clone().unwrap_or_else(|| "ec2-user".to_owned()),
964                    internal_ip,
965                    external_ip,
966                })
967            })
968            .clone()
969    }
970
971    fn strategy_as_server<'a>(
972        &'a self,
973        client_host: &dyn Host,
974        network_hint: PortNetworkHint,
975    ) -> Result<(ClientStrategy<'a>, HostStrategyGetter)> {
976        if matches!(network_hint, PortNetworkHint::Auto)
977            && client_host.can_connect_to(ClientStrategy::UnixSocket(self.id))
978        {
979            Ok((
980                ClientStrategy::UnixSocket(self.id),
981                Box::new(|_| BaseServerStrategy::UnixSocket),
982            ))
983        } else if matches!(
984            network_hint,
985            PortNetworkHint::Auto | PortNetworkHint::TcpPort(_)
986        ) && client_host.can_connect_to(ClientStrategy::InternalTcpPort(self))
987        {
988            Ok((
989                ClientStrategy::InternalTcpPort(self),
990                Box::new(move |_| {
991                    BaseServerStrategy::InternalTcpPort(match network_hint {
992                        PortNetworkHint::Auto => None,
993                        PortNetworkHint::TcpPort(port) => port,
994                    })
995                }),
996            ))
997        } else if matches!(network_hint, PortNetworkHint::Auto)
998            && client_host.can_connect_to(ClientStrategy::ForwardedTcpPort(self))
999        {
1000            Ok((
1001                ClientStrategy::ForwardedTcpPort(self),
1002                Box::new(|me| {
1003                    me.downcast_ref::<AwsEc2Host>()
1004                        .unwrap()
1005                        .request_port_base(&BaseServerStrategy::ExternalTcpPort(22));
1006                    BaseServerStrategy::InternalTcpPort(None)
1007                }),
1008            ))
1009        } else {
1010            anyhow::bail!("Could not find a strategy to connect to AWS EC2 instance")
1011        }
1012    }
1013
1014    fn can_connect_to(&self, typ: ClientStrategy) -> bool {
1015        match typ {
1016            ClientStrategy::UnixSocket(id) => {
1017                #[cfg(unix)]
1018                {
1019                    self.id == id
1020                }
1021
1022                #[cfg(not(unix))]
1023                {
1024                    let _ = id;
1025                    false
1026                }
1027            }
1028            ClientStrategy::InternalTcpPort(target_host) => {
1029                if let Some(aws_target) = <dyn Any>::downcast_ref::<AwsEc2Host>(target_host) {
1030                    self.region == aws_target.region
1031                        && Arc::ptr_eq(&self.network, &aws_target.network)
1032                } else {
1033                    false
1034                }
1035            }
1036            ClientStrategy::ForwardedTcpPort(_) => false,
1037        }
1038    }
1039}