Convenient piping for multiple processes?

Question

Is there a convenient way to pipe processes into each other? (without relying on the shell as this won't always work predictably on WIN32 - I'd assume).

e.g.

(call-process-pipe-chain
  '("ls" "-1")
  '("grep" "-i" "SOME_TEXT")
  '("sort")
  :output "/path/to/file.txt")

Which would be the equivalent of this command in bash:

ls -1 | grep -i SOME_TEXT | sort > /path/to/file.txt

---

• Non-zero exit codes for any of the processes would error or warn.

• Ideally :output could be a string (for a path), buffer - to write into a buffer, t to return as a string or nil to ignore entirely.

Answer 1

This utility function takes list arguments, each one is a commend that is piped to the next command.

• :input keyword argument for input from buffer/string (or nil for none).
• :output keyword argument for output to buffer/file-path (or nil for none).
• Non-zero exit codes error immediately.

---

Example:

;; Outputs upper case Emacs version text.
(call-process-pipe-chain
  '("emacs" "--version")
  '("awk" "{print toupper($0)}")
  :output "/out/file.txt")

---

(defun call-process-pipe-chain (&rest args)
  "Call a chain of commands, each argument must be a list of strings.

Additional keyword arguments may also be passed in.

:input - is used to pipe input into the first commands standard-input.
- nil: no input is passed in.
- string: text to be passed to the standard input.
- buffer: buffer to be passed to the standard input

:output - is used as the target for the final commands output.
- nil: output is ignored.
- t: output is returned as a string.
- string: output is written to file-name.
- buffer: output is written to the buffer."
  (let
    ( ;; To check if this is the first time executing.
      (is-first t)

      ;; Keywords.
      (output nil)
      (input nil)

      ;; Iteration vars.
      (buf-src nil)
      (buf-dst nil))

    ;; Parse keywords.
    (let ((args-no-keywords nil))
      (while args
        (let ((arg-current (car args)))
          (setq args (cdr args))
          (cond
            ((symbolp arg-current)
              (unless args
                (error "Keyword argument %S has no value!" arg-current))
              (let ((arg-value (car args)))
                (setq args (cdr args))
                (pcase arg-current
                  (:input
                    (setq input arg-value)
                    (cond
                      ((null input)) ;; No input.
                      ((eq input t)) ;; String input (standard input data).
                      ((stringp input))
                      ((bufferp input)
                        (unless (buffer-live-p input)
                          (error "Input buffer is invalid %S" input)))
                      (t
                        (error "Input expected a buffer, string or nil."))))
                  (:output
                    (setq output arg-value)
                    (cond
                      ((null output)) ;; No output.
                      ((eq output t)) ;; String output (file path).
                      ((stringp output)
                        (when (string-equal output "")
                          (error "Empty string used as file-path")))
                      ((bufferp output)
                        (unless (buffer-live-p output)
                          (error "Output buffer is invalid %S" output)))

                      (t
                        (error "Output expected a buffer, string or nil."))))

                  (_ (error "Unknown argument %S" arg-current)))))
            ((listp arg-current)
              (push arg-current args-no-keywords))
            (t
              (error
                "Arguments must be property pairs or lists of strings, found %S"
                (type-of arg-current))))))

      (setq args (reverse args-no-keywords)))

    ;; Setup two temporary buffers for source and destination,
    ;; looping over arguments, executing and piping contents.
    (with-temp-buffer
      (setq buf-src (current-buffer))
      (with-temp-buffer
        (setq buf-dst (current-buffer))

        ;; Loop over commands.
        (while args
          (let ((arg-current (car args)))
            (setq args (cdr args))
            (with-current-buffer buf-dst (erase-buffer))
            (let*
              (
                (sentinel-called nil)
                (proc
                  (make-process
                    :name "call-process-pipe-chain"
                    ;; Write to the intermediate buffer or the final output.
                    :buffer
                    (cond
                      ;; Last command, use output.
                      ((and (null args) (bufferp output))
                        output)
                      (t
                        buf-dst))
                    :connection-type 'pipe
                    :command arg-current
                    :sentinel (lambda (_proc _msg) (setq sentinel-called t)))))

              (cond
                ;; Initially, we can only use the input argument.
                (is-first
                  (cond
                    ((null input))
                    ((bufferp input)
                      (with-current-buffer input
                        (process-send-region proc (point-min) (point-max))))
                    ((stringp input)
                      (process-send-string proc input))))
                (t
                  (with-current-buffer buf-src
                    (process-send-region proc (point-min) (point-max))
                    (erase-buffer))))

              (process-send-eof proc)
              (while (not sentinel-called)
                (accept-process-output))

              ;; Check exit-code.
              (let ((exit-code (process-exit-status proc)))
                (unless (zerop exit-code)
                  (error "Command exited code=%d: %S" exit-code arg-current))))

            ;; Swap source/destination buffers.
            (setq buf-src
              (prog1 buf-dst
                (setq buf-dst buf-src)))

            (setq is-first nil)))

        ;; Return the result.
        (cond
          ;; Ignore output.
          ((null output))
          ;; Already written into.
          ((bufferp output))
          ;; Write to the output as a file-path.
          ((stringp output)
            (with-current-buffer buf-src (write-region nil nil output nil 0)))
          ;; Return the output as a string.
          ((eq output t)
            ;; Since the argument was swapped, extract from the 'source'.
            (with-current-buffer buf-src
              (buffer-substring-no-properties (point-min) (point-max)))))))))```