serial: Call set_custom_baudrate() thrice

Call the function before tcsetattr() settings are known, then again
with settings prepared but not yet applied and finally a third time
after tcsetattr().

Darwin support needs this change; there custom_baud code must be
called to modify the settings passed to tcsetattr() and then again
after tcsetattr() returns.

The change should be non-functional on all currently supported systems;
current code calls set_custom_baudrate() before any tcsetattr()
settings are prepared, so we have three stages in total.

This change originates from discussion of the macOS patch proposed by
Denis Ahrens in https://review.coreboot.org/c/flashrom/+/67822

Change-Id: I40cc443cfb7bf6b212b31826d437b898cc13c427
Signed-off-by: Peter Stuge <peter@stuge.se>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/70569
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Thomas Heijligen <src@posteo.de>
---
M custom_baud.c
M custom_baud_linux.c
M include/custom_baud.h
M serial.c
4 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/custom_baud.c b/custom_baud.c
index 28f182c..8bbe6cc 100644
--- a/custom_baud.c
+++ b/custom_baud.c
@@ -19,7 +19,7 @@
 #include "custom_baud.h"
 
 /* Stub, should not get called. */
-int set_custom_baudrate(int fd, unsigned int baud)
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted)
 {
 	errno = ENOSYS; /* Hoping "Function not supported" will make you look here. */
 	return -1;
diff --git a/custom_baud_linux.c b/custom_baud_linux.c
index 2d5f261..761d496 100644
--- a/custom_baud_linux.c
+++ b/custom_baud_linux.c
@@ -29,9 +29,13 @@
  * for more info.
  */
 
-int set_custom_baudrate(int fd, unsigned int baud)
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted)
 {
 	struct termios2 tio;
+
+	if (stage != BEFORE_FLAGS)
+		return 0;
+
 	if (ioctl(fd, TCGETS2, &tio)) {
 		return -1;
 	}
diff --git a/include/custom_baud.h b/include/custom_baud.h
index c8b8fc2..38e6cfc 100644
--- a/include/custom_baud.h
+++ b/include/custom_baud.h
@@ -22,7 +22,13 @@
 	unsigned int baud;
 };
 
-int set_custom_baudrate(int fd, unsigned int baud);
+enum custom_baud_stage {
+	BEFORE_FLAGS = 0,
+	WITH_FLAGS,
+	AFTER_FLAGS
+};
+
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted);
 
 /* Returns 1 if non-exact rate would be used, and setting a custom rate is supported.
    The baudtable must be in ascending order and terminated with a 0-baud entry. */
diff --git a/serial.c b/serial.c
index a0ef632..b9ce2b3 100644
--- a/serial.c
+++ b/serial.c
@@ -179,6 +179,7 @@
 	}
 	msg_pdbg("Baud rate is %ld.\n", dcb.BaudRate);
 #else
+	int custom_baud = (baud >= 0 && use_custom_baud(baud, sp_baudtable));
 	struct termios wanted, observed;
 	if (tcgetattr(fd, &observed) != 0) {
 		msg_perr_strerror("Could not fetch original serial port configuration: ");
@@ -186,8 +187,8 @@
 	}
 	wanted = observed;
 	if (baud >= 0) {
-		if (use_custom_baud(baud, sp_baudtable)) {
-			if (set_custom_baudrate(fd, baud)) {
+		if (custom_baud) {
+			if (set_custom_baudrate(fd, baud, BEFORE_FLAGS, NULL)) {
 				msg_perr_strerror("Could not set custom baudrate: ");
 				return 1;
 			}
@@ -198,7 +199,6 @@
 				msg_perr_strerror("Could not fetch serial port configuration: ");
 				return 1;
 			}
-			msg_pdbg("Using custom baud rate.\n");
 		} else {
 			const struct baudentry *entry = round_baud(baud);
 			if (cfsetispeed(&wanted, entry->flag) != 0 || cfsetospeed(&wanted, entry->flag) != 0) {
@@ -212,6 +212,10 @@
 	wanted.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG | IEXTEN);
 	wanted.c_iflag &= ~(IXON | IXOFF | IXANY | ICRNL | IGNCR | INLCR);
 	wanted.c_oflag &= ~OPOST;
+	if (custom_baud && set_custom_baudrate(fd, baud, WITH_FLAGS, &wanted)) {
+		msg_perr_strerror("Could not set custom baudrate: ");
+		return 1;
+	}
 	if (tcsetattr(fd, TCSANOW, &wanted) != 0) {
 		msg_perr_strerror("Could not change serial port configuration: ");
 		return 1;
@@ -236,6 +240,13 @@
 			 (long)observed.c_oflag, (long)wanted.c_oflag
 			);
 	}
+	if (custom_baud) {
+		if (set_custom_baudrate(fd, baud, AFTER_FLAGS, &wanted)) {
+			msg_perr_strerror("Could not set custom baudrate: ");
+			return 1;
+		}
+		msg_pdbg("Using custom baud rate.\n");
+	}
 	if (cfgetispeed(&observed) != cfgetispeed(&wanted) ||
 	    cfgetospeed(&observed) != cfgetospeed(&wanted)) {
 		msg_pwarn("Could not set baud rates exactly.\n");

